def setUp(self):
''' Simple engine without meta - engine_A
'''
path_data = 'tests/engine_A_data.csv'
data = pd.read_csv(path_data)
# Setup engine_A
self.engine_A = WeightEngine(data=data)
self.scheme_name_A1 = 'scheme_name_A1'
self.scheme_name_A2 = 'scheme_name_A2'
self.scheme_name_A3 = 'scheme_name_A3'
# Setup schemes to use in tests
self.scheme_A1 = Rim(self.scheme_name_A1)
self.scheme_A1.lists = ['column1', 'column2']
self.scheme_A1.add_group(name='Senior Type 1', filter='column3==1',
targets={
'column1': [32.00, 31.00, 37.00],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 2', filter='column3==1',
targets={
'column1': [33.40, 33.40, 33.20],
'column2': [11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11]
})
self.scheme_A1.add_group(name='Senior Type 3', filter='column3==3',
targets={
'column1': [33.2, 29.7, 37.1],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 4', filter='column3==4',
targets={
'column1': [33.2, 29.7, 37.1],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34, 12.00]
})
self.scheme_A2 = Rim(self.scheme_name_A2)
self.scheme_A2.lists = ['column1', 'column2']
self.scheme_A2.add_group(name='Senior Type 1', filter='column3==1',
targets={
'column1': [37.00, 32.00, 31.00],
'column2': [13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 2', filter='column3==1',
targets={
'column1': [33.2, 33.40, 33.40],
'column2': [11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11]
})
self.scheme_A2.add_group(name='Senior Type 3', filter='column3==3',
targets={
'column1': [37.1, 33.2, 29.7],
'column2': [13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 4', filter='column3==4',
targets={
'column1': [37.1, 33.2, 29.7],
'column2': [12.00, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34]
})
self.scheme_A3 = Rim(self.scheme_name_A3)
self.scheme_A3.lists = ['profile_gender']
self.scheme_A3.targets = {'profile_gender' : [47, 53]}
self.scheme_A3.add_group(
name='11-19', filter='age_group=2', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='31-39', filter='age_group=4', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='41-49', filter='age_group=5', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='51-59', filter='age_group=6', targets=self.scheme_A3.targets
)
self.scheme_A3.group_targets({
'11-19': 25,
'31-39': 25,
'41-49': 25,
'51-59': 25
})
''' Complex engine with meta - engine_B
'''
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
self.scheme_name_B1 = 'scheme_name_B1'
engine_B = WeightEngine(data=data, meta=meta)
# Setup schemes to use in tests
self.scheme_B1 = Rim(self.scheme_name_B1)
self.scheme_B1.lists = ['profile_gender', 'age_group']
class TestScheme(unittest.TestCase):
def setUp(self):
self.analog_scheme = Rim('analog')
def test_constructor(self):
#Check to see if everything initialised correctly
self.assertIsInstance(self.analog_scheme, Rim)
self.assertEqual(self.analog_scheme.name, 'analog')
self.assertEqual(self.analog_scheme.lists, [])
self.assertEqual(self.analog_scheme.max_iterations, 1000)
self.assertEqual(self.analog_scheme.convcrit, 0.01)
self.assertEqual(self.analog_scheme.cap, 0)
self.assertTrue(self.analog_scheme.dropna)
self.assertIsInstance(self.analog_scheme._impute_method_specific, dict)
self.assertIsNone(self.analog_scheme.weight_column_name)
self.assertEqual(self.analog_scheme.total, 0)
def test_renamefunc(self):
# Test the rename function
self.analog_scheme.lists = ['gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04']
# This command should throw (raise) an exception not print to stdout (Maybe!)
with self.assertRaises(ValueError):
self.analog_scheme.rename_list(find='somethingnotinthelist', replace='With_this')
rename_successfull = self.analog_scheme.rename_list(find='gender', replace='sex')
self.assertTrue(rename_successfull)
self.assertIn('sex', self.analog_scheme.lists)
def test_cap(self):
# Test the cap functionality
self.assertEqual(0, self.analog_scheme.cap)
self.assertFalse(self.analog_scheme.use_cap())
self.analog_scheme.cap = 5
self.assertEqual(5, self.analog_scheme.cap)
self.assertTrue(self.analog_scheme.use_cap())
#Check to see if it reverts properly to not using cap
self.analog_scheme.cap = 0
self.assertFalse(self.analog_scheme.use_cap())
def test_groups(self):
scheme = Rim('digital')
self.assertIsInstance(scheme.groups, dict)
self.assertIsNone(scheme.groups['__default_group_name__']['report'])
self.assertIsNone(scheme.groups['__default_group_name__']['filters'])
scheme.lists = ['gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04']
scheme.add_group(name='Apple', filter='ownership==1')
self.assertIn('Apple', scheme.groups)
scheme.add_group(name='Samsung', filter={'ownership': 2})
self.assertIn('Samsung', scheme.groups)
# Try an invalid filter
# It should not contain any filters since it was called incorrectly
with self.assertRaises(Exception):
scheme.add_group(name='Motorola', filter='ownership3')
# Check to see if the available methods to add filter are equal
self.assertEqual(scheme.groups['Samsung']['filters']['ownership'], 2)
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 1)
# Test for changes to valid filter
scheme.group_filter(group_name='Apple', filter='ownership==8')
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 8)
# Test for changes to an non-existing filter, NOTE: the group_filter function actually creates the group motorola
scheme.group_filter(group_name='Motorola', filter='ownership==12')
self.assertEqual(scheme.groups['Motorola']['filters']['ownership'], 12)
#Test for invalid filter type
#Test for invalid group with dictionary
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Motorola', filter=[])
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Ferrari', filter={'ownership': 3})
# Try to change a filter for invalid group
self.assertNotIn('doesnotexist', scheme.groups.keys())
# Try to make illegal changes to filter in a valid group
self.assertNotEqual(scheme.groups['Apple']['filters']['ownership'], 1)
with self.assertRaises(Exception):
scheme.group_filter(group_name='Apple', filter='ownership1')
#The targets should be empty lists
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], [])
#Test for incorrect target change
with self.assertRaises(ValueError):
scheme.set_targets(group_name='Apple', targets={'gender': 1234})
#Test for setting targets to a list that wasn't given
# with self.assertRaises(ValueError):
# scheme.set_targets(group_name='Apple', targets={'doesnotexist': [80, 20]})
#Set valid targets
valid_targets={
'gender': [50, 50],
'column1c': [20, 18, 25, 21, 16],
'q04': [20, 55, 12.5, 12.5],
'sta_wo': [50, 50],
'abschluss': [60, 40],
'q06': [20, 20, 20, 20, 20]
}
scheme.set_targets(group_name='Apple', targets=valid_targets)
#Test that only the most recently set targets are still in the scheme
self.assertNotIn('doesnotexist', scheme.groups['Apple']['targets'].keys())
#Test that the targets were applied to the lists corrected
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], valid_targets[key])
#add group_targets
scheme.group_targets(
{
"Apple": 30,
"Samsung": 40,
"Motorola": 30
}
)
self.assertItemsEqual(scheme._group_targets.keys(), ['Motorola', 'Apple', 'Samsung'])
self.assertItemsEqual(scheme._group_targets.values(), [0.3, 0.3, 0.4])
class TestEngine(unittest.TestCase):
def setUp(self):
''' Simple engine without meta - engine_A
'''
path_data = 'tests/engine_A_data.csv'
data = pd.read_csv(path_data)
# Setup engine_A
self.engine_A = WeightEngine(data=data)
self.scheme_name_A1 = 'scheme_name_A1'
self.scheme_name_A2 = 'scheme_name_A2'
self.scheme_name_A3 = 'scheme_name_A3'
# Setup schemes to use in tests
self.scheme_A1 = Rim(self.scheme_name_A1)
self.scheme_A1.lists = ['column1', 'column2']
self.scheme_A1.add_group(name='Senior Type 1', filter='column3==1',
targets={
'column1': [32.00, 31.00, 37.00],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 2', filter='column3==1',
targets={
'column1': [33.40, 33.40, 33.20],
'column2': [11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11]
})
self.scheme_A1.add_group(name='Senior Type 3', filter='column3==3',
targets={
'column1': [33.2, 29.7, 37.1],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 4', filter='column3==4',
targets={
'column1': [33.2, 29.7, 37.1],
'column2': [23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34, 12.00]
})
self.scheme_A2 = Rim(self.scheme_name_A2)
self.scheme_A2.lists = ['column1', 'column2']
self.scheme_A2.add_group(name='Senior Type 1', filter='column3==1',
targets={
'column1': [37.00, 32.00, 31.00],
'column2': [13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 2', filter='column3==1',
targets={
'column1': [33.2, 33.40, 33.40],
'column2': [11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11, 11.11]
})
self.scheme_A2.add_group(name='Senior Type 3', filter='column3==3',
targets={
'column1': [37.1, 33.2, 29.7],
'column2': [13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 4', filter='column3==4',
targets={
'column1': [37.1, 33.2, 29.7],
'column2': [12.00, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34]
})
self.scheme_A3 = Rim(self.scheme_name_A3)
self.scheme_A3.lists = ['profile_gender']
self.scheme_A3.targets = {'profile_gender' : [47, 53]}
self.scheme_A3.add_group(
name='11-19', filter='age_group=2', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='31-39', filter='age_group=4', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='41-49', filter='age_group=5', targets=self.scheme_A3.targets
)
self.scheme_A3.add_group(
name='51-59', filter='age_group=6', targets=self.scheme_A3.targets
)
self.scheme_A3.group_targets({
'11-19': 25,
'31-39': 25,
'41-49': 25,
'51-59': 25
})
''' Complex engine with meta - engine_B
'''
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
self.scheme_name_B1 = 'scheme_name_B1'
engine_B = WeightEngine(data=data, meta=meta)
# Setup schemes to use in tests
self.scheme_B1 = Rim(self.scheme_name_B1)
self.scheme_B1.lists = ['profile_gender', 'age_group']
# self.scheme_B1.set_targets()
def test_constructor(self):
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
engine_B = WeightEngine(data=data, meta=meta)
self.assertIsNotNone(engine_B._df)
self.assertTrue(engine_B.dropna)
self.assertEqual(engine_B.schemes, {})
self.assertIsInstance(engine_B.schemes, dict)
def test_add_scheme_and_dataframe(self):
#A list of scheme names used in setUp used for comparison
scheme_names = [self.scheme_name_A1, self.scheme_name_A2]
self.engine_A.add_scheme(scheme=self.scheme_A2, key='identity')
# Should now contain a dict with scheme_name_A2 as the first key
self.assertEqual(self.engine_A.schemes.keys()[0], self.scheme_name_A2)
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
# Should now contain a dict with scheme_name_A2 and scheme_name_A1 as keys
for key in self.engine_A.schemes:
self.assertIn(key, scheme_names)
self.assertIn('identity', self.engine_A.schemes[key]['key'])
# Sets weights_scheme_name_A1 and weights_scheme_name_A2 to ones
self.engine_A._df[self.scheme_A1.weight_name()] = pd.np.ones(len(self.engine_A._df))
self.engine_A._df[self.scheme_A2.weight_name()] = pd.np.ones(len(self.engine_A._df))
for key in self.engine_A.schemes:
weight_scheme = self.engine_A._df['weights_'+key]
boolean_vector = (weight_scheme == pd.np.ones(len(weight_scheme)))
self.assertTrue(boolean_vector.all())
self.engine_A.run(schemes=[key])
boolean_vector = (weight_scheme == pd.np.ones(len(weight_scheme)))
self.assertFalse(boolean_vector.all())
def test_add_scheme_no_key(self):
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
self.assertIsNotNone(self.engine_A.schemes[self.scheme_name_A1]['key'])
def test_weight_lazy(self):
return
self.engine_A.add_scheme(scheme=self.scheme_A2, key='identity')
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
self.assertNotIn('weights_scheme_name_A2', self.engine_A._df.columns)
self.engine_A.weight()
self.assertIn('weights_%s' % self.scheme_name_A1, self.engine_A._df.columns)
self.assertIn('weights_%s' % self.scheme_name_A2, self.engine_A._df.columns)
def test_group_targets(self):
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
weight = '_'.join(
['weights',
self.scheme_name_A3]
)
# Run weights for scheme_A3
engine_B = WeightEngine(data=data, meta=meta)
engine_B.add_scheme(scheme=self.scheme_A3, key='identity')
engine_B.run()
data_A3 = engine_B.dataframe("scheme_name_A3")
# check identical weighted column frequencies
df = data_A3.pivot_table(
values=[weight],
index=['profile_gender'],
columns=['age_group'],
aggfunc='sum'
)
for column in df.columns.tolist():
self.assertTrue(
numpy.allclose(df[column].values, numpy.array([1.645, 1.855]))
)
#check the weight column counts & sum are equal to index length (14)
a = numpy.asscalar(data_A3[weight].count())
b = numpy.asscalar(data_A3[weight].sum())
c = data_A3.shape[0]
self.assertTrue(int(a) == int(b) == int(c))
# check weighted group frequencies have euqal proportions
values = data_A3.pivot_table(
values=[weight],
index=['age_group'],
aggfunc='sum'
).values
self.assertTrue(numpy.allclose(values, 3.5))
def setUp(self):
self.analog_scheme = Rim('analog')
def test_groups(self):
scheme = Rim('digital')
self.assertIsInstance(scheme.groups, dict)
self.assertIsNone(scheme.groups['__default_group_name__']['report'])
self.assertIsNone(scheme.groups['__default_group_name__']['filters'])
scheme.lists = ['gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04']
scheme.add_group(name='Apple', filter='ownership==1')
self.assertIn('Apple', scheme.groups)
scheme.add_group(name='Samsung', filter={'ownership': 2})
self.assertIn('Samsung', scheme.groups)
# Try an invalid filter
# It should not contain any filters since it was called incorrectly
with self.assertRaises(Exception):
scheme.add_group(name='Motorola', filter='ownership3')
# Check to see if the available methods to add filter are equal
self.assertEqual(scheme.groups['Samsung']['filters']['ownership'], 2)
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 1)
# Test for changes to valid filter
scheme.group_filter(group_name='Apple', filter='ownership==8')
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 8)
# Test for changes to an non-existing filter, NOTE: the group_filter function actually creates the group motorola
scheme.group_filter(group_name='Motorola', filter='ownership==12')
self.assertEqual(scheme.groups['Motorola']['filters']['ownership'], 12)
#Test for invalid filter type
#Test for invalid group with dictionary
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Motorola', filter=[])
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Ferrari', filter={'ownership': 3})
# Try to change a filter for invalid group
self.assertNotIn('doesnotexist', scheme.groups.keys())
# Try to make illegal changes to filter in a valid group
self.assertNotEqual(scheme.groups['Apple']['filters']['ownership'], 1)
with self.assertRaises(Exception):
scheme.group_filter(group_name='Apple', filter='ownership1')
#The targets should be empty lists
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], [])
#Test for incorrect target change
with self.assertRaises(ValueError):
scheme.set_targets(group_name='Apple', targets={'gender': 1234})
#Test for setting targets to a list that wasn't given
# with self.assertRaises(ValueError):
# scheme.set_targets(group_name='Apple', targets={'doesnotexist': [80, 20]})
#Set valid targets
valid_targets={
'gender': [50, 50],
'column1c': [20, 18, 25, 21, 16],
'q04': [20, 55, 12.5, 12.5],
'sta_wo': [50, 50],
'abschluss': [60, 40],
'q06': [20, 20, 20, 20, 20]
}
scheme.set_targets(group_name='Apple', targets=valid_targets)
#Test that only the most recently set targets are still in the scheme
self.assertNotIn('doesnotexist', scheme.groups['Apple']['targets'].keys())
#Test that the targets were applied to the lists corrected
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], valid_targets[key])
#add group_targets
scheme.group_targets(
{
"Apple": 30,
"Samsung": 40,
"Motorola": 30
}
)
self.assertItemsEqual(scheme._group_targets.keys(), ['Motorola', 'Apple', 'Samsung'])
self.assertItemsEqual(scheme._group_targets.values(), [0.3, 0.3, 0.4])
class TestScheme(unittest.TestCase):
def setUp(self):
self.analog_scheme = Rim('analog')
def test_constructor(self):
#Check to see if everything initialised correctly
self.assertIsInstance(self.analog_scheme, Rim)
self.assertEqual(self.analog_scheme.name, 'analog')
self.assertEqual(self.analog_scheme.lists, [])
self.assertEqual(self.analog_scheme.max_iterations, 1000)
self.assertEqual(self.analog_scheme.convcrit, 0.01)
self.assertEqual(self.analog_scheme.cap, 0)
self.assertTrue(self.analog_scheme.dropna)
self.assertIsInstance(self.analog_scheme._impute_method_specific, dict)
self.assertIsNone(self.analog_scheme.weight_column_name)
self.assertEqual(self.analog_scheme.total, 0)
def test_renamefunc(self):
# Test the rename function
self.analog_scheme.lists = [
'gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04'
]
# This command should throw (raise) an exception not print to stdout (Maybe!)
with self.assertRaises(ValueError):
self.analog_scheme.rename_list(find='somethingnotinthelist',
replace='With_this')
rename_successfull = self.analog_scheme.rename_list(find='gender',
replace='sex')
self.assertTrue(rename_successfull)
self.assertIn('sex', self.analog_scheme.lists)
def test_cap(self):
# Test the cap functionality
self.assertEqual(0, self.analog_scheme.cap)
self.assertFalse(self.analog_scheme.use_cap())
self.analog_scheme.cap = 5
self.assertEqual(5, self.analog_scheme.cap)
self.assertTrue(self.analog_scheme.use_cap())
#Check to see if it reverts properly to not using cap
self.analog_scheme.cap = 0
self.assertFalse(self.analog_scheme.use_cap())
def test_groups(self):
scheme = Rim('digital')
self.assertIsInstance(scheme.groups, dict)
self.assertIsNone(scheme.groups['__default_group_name__']['report'])
self.assertIsNone(scheme.groups['__default_group_name__']['filters'])
scheme.lists = [
'gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04'
]
scheme.add_group(name='Apple', filter='ownership==1')
self.assertIn('Apple', scheme.groups)
scheme.add_group(name='Samsung', filter={'ownership': 2})
self.assertIn('Samsung', scheme.groups)
# Try an invalid filter
# It should not contain any filters since it was called incorrectly
with self.assertRaises(Exception):
scheme.add_group(name='Motorola', filter='ownership3')
# Check to see if the available methods to add filter are equal
self.assertEqual(scheme.groups['Samsung']['filters']['ownership'], 2)
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 1)
# Test for changes to valid filter
scheme.group_filter(group_name='Apple', filter='ownership==8')
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 8)
# Test for changes to an non-existing filter, NOTE: the group_filter function actually creates the group motorola
scheme.group_filter(group_name='Motorola', filter='ownership==12')
self.assertEqual(scheme.groups['Motorola']['filters']['ownership'], 12)
#Test for invalid filter type
#Test for invalid group with dictionary
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Motorola', filter=[])
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Ferrari', filter={'ownership': 3})
# Try to change a filter for invalid group
self.assertNotIn('doesnotexist', scheme.groups.keys())
# Try to make illegal changes to filter in a valid group
self.assertNotEqual(scheme.groups['Apple']['filters']['ownership'], 1)
with self.assertRaises(Exception):
scheme.group_filter(group_name='Apple', filter='ownership1')
#The targets should be empty lists
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], [])
#Test for incorrect target change
with self.assertRaises(ValueError):
scheme.set_targets(group_name='Apple', targets={'gender': 1234})
#Test for setting targets to a list that wasn't given
# with self.assertRaises(ValueError):
# scheme.set_targets(group_name='Apple', targets={'doesnotexist': [80, 20]})
#Set valid targets
valid_targets = {
'gender': [50, 50],
'column1c': [20, 18, 25, 21, 16],
'q04': [20, 55, 12.5, 12.5],
'sta_wo': [50, 50],
'abschluss': [60, 40],
'q06': [20, 20, 20, 20, 20]
}
scheme.set_targets(group_name='Apple', targets=valid_targets)
#Test that only the most recently set targets are still in the scheme
self.assertNotIn('doesnotexist',
scheme.groups['Apple']['targets'].keys())
#Test that the targets were applied to the lists corrected
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key],
valid_targets[key])
#add group_targets
scheme.group_targets({"Apple": 30, "Samsung": 40, "Motorola": 30})
self.assertItemsEqual(scheme._group_targets.keys(),
['Motorola', 'Apple', 'Samsung'])
self.assertItemsEqual(scheme._group_targets.values(), [0.3, 0.3, 0.4])
def test_groups(self):
scheme = Rim('digital')
self.assertIsInstance(scheme.groups, dict)
self.assertIsNone(scheme.groups['__default_group_name__']['report'])
self.assertIsNone(scheme.groups['__default_group_name__']['filters'])
scheme.lists = [
'gender', 'column1c', 'q06', 'sta_wo', 'abschluss', 'q04'
]
scheme.add_group(name='Apple', filter='ownership==1')
self.assertIn('Apple', scheme.groups)
scheme.add_group(name='Samsung', filter={'ownership': 2})
self.assertIn('Samsung', scheme.groups)
# Try an invalid filter
# It should not contain any filters since it was called incorrectly
with self.assertRaises(Exception):
scheme.add_group(name='Motorola', filter='ownership3')
# Check to see if the available methods to add filter are equal
self.assertEqual(scheme.groups['Samsung']['filters']['ownership'], 2)
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 1)
# Test for changes to valid filter
scheme.group_filter(group_name='Apple', filter='ownership==8')
self.assertEqual(scheme.groups['Apple']['filters']['ownership'], 8)
# Test for changes to an non-existing filter, NOTE: the group_filter function actually creates the group motorola
scheme.group_filter(group_name='Motorola', filter='ownership==12')
self.assertEqual(scheme.groups['Motorola']['filters']['ownership'], 12)
#Test for invalid filter type
#Test for invalid group with dictionary
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Motorola', filter=[])
with self.assertRaises(ValueError):
scheme.group_filter(group_name='Ferrari', filter={'ownership': 3})
# Try to change a filter for invalid group
self.assertNotIn('doesnotexist', scheme.groups.keys())
# Try to make illegal changes to filter in a valid group
self.assertNotEqual(scheme.groups['Apple']['filters']['ownership'], 1)
with self.assertRaises(Exception):
scheme.group_filter(group_name='Apple', filter='ownership1')
#The targets should be empty lists
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key], [])
#Test for incorrect target change
with self.assertRaises(ValueError):
scheme.set_targets(group_name='Apple', targets={'gender': 1234})
#Test for setting targets to a list that wasn't given
# with self.assertRaises(ValueError):
# scheme.set_targets(group_name='Apple', targets={'doesnotexist': [80, 20]})
#Set valid targets
valid_targets = {
'gender': [50, 50],
'column1c': [20, 18, 25, 21, 16],
'q04': [20, 55, 12.5, 12.5],
'sta_wo': [50, 50],
'abschluss': [60, 40],
'q06': [20, 20, 20, 20, 20]
}
scheme.set_targets(group_name='Apple', targets=valid_targets)
#Test that only the most recently set targets are still in the scheme
self.assertNotIn('doesnotexist',
scheme.groups['Apple']['targets'].keys())
#Test that the targets were applied to the lists corrected
for key in scheme.groups['Apple']['targets']:
self.assertEqual(scheme.groups['Apple']['targets'][key],
valid_targets[key])
#add group_targets
scheme.group_targets({"Apple": 30, "Samsung": 40, "Motorola": 30})
self.assertItemsEqual(scheme._group_targets.keys(),
['Motorola', 'Apple', 'Samsung'])
self.assertItemsEqual(scheme._group_targets.values(), [0.3, 0.3, 0.4])
def setUp(self):
self.analog_scheme = Rim('analog')
def setUp(self):
''' Simple engine without meta - engine_A
'''
path_data = 'tests/engine_A_data.csv'
data = pd.read_csv(path_data)
# Setup engine_A
self.engine_A = WeightEngine(data=data)
self.scheme_name_A1 = 'scheme_name_A1'
self.scheme_name_A2 = 'scheme_name_A2'
self.scheme_name_A3 = 'scheme_name_A3'
# Setup schemes to use in tests
self.scheme_A1 = Rim(self.scheme_name_A1)
self.scheme_A1.lists = ['column1', 'column2']
self.scheme_A1.add_group(
name='Senior Type 1',
filter='column3==1',
targets={
'column1': [32.00, 31.00, 37.00],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 2',
filter='column3==1',
targets={
'column1': [33.40, 33.40, 33.20],
'column2': [
11.11, 11.11, 11.11, 11.11, 11.11,
11.11, 11.11, 11.11, 11.11
]
})
self.scheme_A1.add_group(
name='Senior Type 3',
filter='column3==3',
targets={
'column1': [33.2, 29.7, 37.1],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(
name='Senior Type 4',
filter='column3==4',
targets={
'column1': [33.2, 29.7, 37.1],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34, 12.00]
})
self.scheme_A2 = Rim(self.scheme_name_A2)
self.scheme_A2.lists = ['column1', 'column2']
self.scheme_A2.add_group(
name='Senior Type 1',
filter='column3==1',
targets={
'column1': [37.00, 32.00, 31.00],
'column2':
[13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 2',
filter='column3==1',
targets={
'column1': [33.2, 33.40, 33.40],
'column2': [
11.11, 11.11, 11.11, 11.11, 11.11,
11.11, 11.11, 11.11, 11.11
]
})
self.scheme_A2.add_group(
name='Senior Type 3',
filter='column3==3',
targets={
'column1': [37.1, 33.2, 29.7],
'column2':
[13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(
name='Senior Type 4',
filter='column3==4',
targets={
'column1': [37.1, 33.2, 29.7],
'column2':
[12.00, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34]
})
self.scheme_A3 = Rim(self.scheme_name_A3)
self.scheme_A3.lists = ['profile_gender']
self.scheme_A3.targets = {'profile_gender': [47, 53]}
self.scheme_A3.add_group(name='11-19',
filter='age_group=2',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='31-39',
filter='age_group=4',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='41-49',
filter='age_group=5',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='51-59',
filter='age_group=6',
targets=self.scheme_A3.targets)
self.scheme_A3.group_targets({
'11-19': 25,
'31-39': 25,
'41-49': 25,
'51-59': 25
})
''' Complex engine with meta - engine_B
'''
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
self.scheme_name_B1 = 'scheme_name_B1'
engine_B = WeightEngine(data=data, meta=meta)
# Setup schemes to use in tests
self.scheme_B1 = Rim(self.scheme_name_B1)
self.scheme_B1.lists = ['profile_gender', 'age_group']
class TestEngine(unittest.TestCase):
def setUp(self):
''' Simple engine without meta - engine_A
'''
path_data = 'tests/engine_A_data.csv'
data = pd.read_csv(path_data)
# Setup engine_A
self.engine_A = WeightEngine(data=data)
self.scheme_name_A1 = 'scheme_name_A1'
self.scheme_name_A2 = 'scheme_name_A2'
self.scheme_name_A3 = 'scheme_name_A3'
# Setup schemes to use in tests
self.scheme_A1 = Rim(self.scheme_name_A1)
self.scheme_A1.lists = ['column1', 'column2']
self.scheme_A1.add_group(
name='Senior Type 1',
filter='column3==1',
targets={
'column1': [32.00, 31.00, 37.00],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(name='Senior Type 2',
filter='column3==1',
targets={
'column1': [33.40, 33.40, 33.20],
'column2': [
11.11, 11.11, 11.11, 11.11, 11.11,
11.11, 11.11, 11.11, 11.11
]
})
self.scheme_A1.add_group(
name='Senior Type 3',
filter='column3==3',
targets={
'column1': [33.2, 29.7, 37.1],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04, 13.3]
})
self.scheme_A1.add_group(
name='Senior Type 4',
filter='column3==4',
targets={
'column1': [33.2, 29.7, 37.1],
'column2':
[23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34, 12.00]
})
self.scheme_A2 = Rim(self.scheme_name_A2)
self.scheme_A2.lists = ['column1', 'column2']
self.scheme_A2.add_group(
name='Senior Type 1',
filter='column3==1',
targets={
'column1': [37.00, 32.00, 31.00],
'column2':
[13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(name='Senior Type 2',
filter='column3==1',
targets={
'column1': [33.2, 33.40, 33.40],
'column2': [
11.11, 11.11, 11.11, 11.11, 11.11,
11.11, 11.11, 11.11, 11.11
]
})
self.scheme_A2.add_group(
name='Senior Type 3',
filter='column3==3',
targets={
'column1': [37.1, 33.2, 29.7],
'column2':
[13.3, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 31.04]
})
self.scheme_A2.add_group(
name='Senior Type 4',
filter='column3==4',
targets={
'column1': [37.1, 33.2, 29.7],
'column2':
[12.00, 23.13, 14.32, 4.78, 4.70, 2.65, 2.61, 3.47, 32.34]
})
self.scheme_A3 = Rim(self.scheme_name_A3)
self.scheme_A3.lists = ['profile_gender']
self.scheme_A3.targets = {'profile_gender': [47, 53]}
self.scheme_A3.add_group(name='11-19',
filter='age_group=2',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='31-39',
filter='age_group=4',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='41-49',
filter='age_group=5',
targets=self.scheme_A3.targets)
self.scheme_A3.add_group(name='51-59',
filter='age_group=6',
targets=self.scheme_A3.targets)
self.scheme_A3.group_targets({
'11-19': 25,
'31-39': 25,
'41-49': 25,
'51-59': 25
})
''' Complex engine with meta - engine_B
'''
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
self.scheme_name_B1 = 'scheme_name_B1'
engine_B = WeightEngine(data=data, meta=meta)
# Setup schemes to use in tests
self.scheme_B1 = Rim(self.scheme_name_B1)
self.scheme_B1.lists = ['profile_gender', 'age_group']
# self.scheme_B1.set_targets()
def test_constructor(self):
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
engine_B = WeightEngine(data=data, meta=meta)
self.assertIsNotNone(engine_B._df)
self.assertTrue(engine_B.dropna)
self.assertEqual(engine_B.schemes, {})
self.assertIsInstance(engine_B.schemes, dict)
def test_add_scheme_and_dataframe(self):
#A list of scheme names used in setUp used for comparison
scheme_names = [self.scheme_name_A1, self.scheme_name_A2]
self.engine_A.add_scheme(scheme=self.scheme_A2, key='identity')
# Should now contain a dict with scheme_name_A2 as the first key
self.assertEqual(self.engine_A.schemes.keys()[0], self.scheme_name_A2)
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
# Should now contain a dict with scheme_name_A2 and scheme_name_A1 as keys
for key in self.engine_A.schemes:
self.assertIn(key, scheme_names)
self.assertIn('identity', self.engine_A.schemes[key]['key'])
# Sets weights_scheme_name_A1 and weights_scheme_name_A2 to ones
self.engine_A._df[self.scheme_A1.weight_name()] = pd.np.ones(
len(self.engine_A._df))
self.engine_A._df[self.scheme_A2.weight_name()] = pd.np.ones(
len(self.engine_A._df))
for key in self.engine_A.schemes:
weight_scheme = self.engine_A._df['weights_' + key]
boolean_vector = (weight_scheme == pd.np.ones(len(weight_scheme)))
self.assertTrue(boolean_vector.all())
self.engine_A.run(schemes=[key])
boolean_vector = (weight_scheme == pd.np.ones(len(weight_scheme)))
self.assertFalse(boolean_vector.all())
def test_add_scheme_no_key(self):
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
self.assertIsNotNone(self.engine_A.schemes[self.scheme_name_A1]['key'])
def test_weight_lazy(self):
return
self.engine_A.add_scheme(scheme=self.scheme_A2, key='identity')
self.engine_A.add_scheme(scheme=self.scheme_A1, key='identity')
self.assertNotIn('weights_scheme_name_A2', self.engine_A._df.columns)
self.engine_A.weight()
self.assertIn('weights_%s' % self.scheme_name_A1,
self.engine_A._df.columns)
self.assertIn('weights_%s' % self.scheme_name_A2,
self.engine_A._df.columns)
def test_group_targets(self):
path_data = 'tests/engine_B_data.csv'
path_meta = 'tests/engine_B_meta.json'
data = pd.read_csv(path_data)
meta = json.load(file(path_meta))
weight = '_'.join(['weights', self.scheme_name_A3])
# Run weights for scheme_A3
engine_B = WeightEngine(data=data, meta=meta)
engine_B.add_scheme(scheme=self.scheme_A3, key='identity')
engine_B.run()
data_A3 = engine_B.dataframe("scheme_name_A3")
# check identical weighted column frequencies
df = data_A3.pivot_table(values=[weight],
index=['profile_gender'],
columns=['age_group'],
aggfunc='sum')
for column in df.columns.tolist():
self.assertTrue(
numpy.allclose(df[column].values, numpy.array([1.645, 1.855])))
#check the weight column counts & sum are equal to index length (14)
a = numpy.asscalar(data_A3[weight].count())
b = numpy.asscalar(data_A3[weight].sum())
c = data_A3.shape[0]
self.assertTrue(int(a) == int(b) == int(c))
# check weighted group frequencies have euqal proportions
values = data_A3.pivot_table(values=[weight],
index=['age_group'],
aggfunc='sum').values
self.assertTrue(numpy.allclose(values, 3.5))
