def _test_calculator(self):
self.dframe = self.dataset.dframe()
columns = self.dframe.columns.tolist()
self.start_num_cols = len(columns)
self.added_num_cols = 0
column_labels_to_slugs = {
column_attrs[Dataset.LABEL]: (column_name) for
(column_name, column_attrs) in self.dataset.schema.items()
}
self.label_list, self.slugified_key_list = [
list(ary) for ary in zip(*column_labels_to_slugs.items())
]
for idx, formula in enumerate(self.calculations):
name = 'test-%s' % idx
self.parser.validate_formula(formula)
calculator = Calculator(self.dataset)
calculation = Calculation()
calculation.save(self.dataset, formula, name, self.group)
calculator.calculate_columns([calculation])
self.column_labels_to_slugs = self.dataset.schema.labels_to_slugs
self._test_calculation_results(name, formula)
def _test_calculator(self):
self.dframe = self.dataset.dframe()
columns = self.dframe.columns.tolist()
self.start_num_cols = len(columns)
self.added_num_cols = 0
column_labels_to_slugs = {
column_attrs[Dataset.LABEL]: (column_name)
for (column_name, column_attrs) in self.dataset.schema.items()
}
self.label_list, self.slugified_key_list = [
list(ary) for ary in zip(*column_labels_to_slugs.items())
]
for idx, formula in enumerate(self.calculations):
name = 'test-%s' % idx
Parser.validate_formula(formula, self.dataset)
calculation = Calculation()
calculation.save(self.dataset, formula, name, self.group)
self.now = now()
calculate_columns(self.dataset, [calculation])
self.column_labels_to_slugs = self.dataset.schema.labels_to_slugs
self._test_calculation_results(name, formula)
def test_disallow_delete_dependent_calculation(self):
record = self._save_observations_and_calculation()
self.name = 'test1'
record = self._save_calculation('test')
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, ['test1'])
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
assert_raises(DependencyError, calculation.delete, self.dataset)
def test_save_improper_formula_no_data(self):
assert_raises(ParseError,
Calculation().save, self.dataset, 'NON_EXISTENT_COLUMN',
self.name)
try:
Calculation().save(self.dataset, 'NON_EXISTENT_COLUMN', self.name)
except ParseError as e:
self.assertTrue('No schema' in e.__str__())
def test_disallow_delete_dependent_calculation(self):
self._save_observations_and_calculation()
self.name = 'test1'
self._save_calculation('test')
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, ['test1'])
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
assert_raises(DependencyError, calculation.delete, self.dataset)
def test_save_unparsable_formula_no_data(self):
assert_raises(ParseError,
Calculation().save, self.dataset, '=NON_EXISTENT_COLUMN',
self.name)
try:
Calculation().save(self.dataset, '=NON_EXISTENT_COLUMN', self.name)
except ParseError as e:
self.assertTrue('Parse Failure' in e.__str__())
def test_removes_dependent_calculations(self):
record = self._save_observations_and_calculation()
self.name = 'test1'
record = self._save_calculation('test')
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, ['test1'])
calculation = Calculation.find_one(self.dataset.dataset_id, 'test1')
calculation.delete(self.dataset)
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, [])
def test_removes_dependent_calculations(self):
self._save_observations_and_calculation()
self.name = 'test1'
self._save_calculation('test')
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, ['test1'])
calculation = Calculation.find_one(self.dataset.dataset_id, 'test1')
calculation.delete(self.dataset)
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, [])
def action(dataset):
if json_file:
calculations = safe_json_loads(json_file.file.read())
Calculation.create_from_list_or_dict(dataset, calculations)
success_message = "created calculations from JSON for dataset" ":%s" % dataset_id
elif formula is None or name is None:
raise ArgumentError("Must provide both formula and name argume" "nts, or json_file argument")
else:
Calculation.create(dataset, formula, name, group)
success_message = "created calculation: %s for dataset: %s" % (name, dataset_id)
return {self.SUCCESS: success_message}
def action(dataset):
if json_file:
calculations = safe_json_loads(json_file.file.read())
Calculation.create_from_list_or_dict(dataset, calculations)
success_message = 'created calculations from JSON'
elif formula is None or name is None:
raise ArgumentError('Must provide both formula and name argume'
'nts, or json_file argument')
else:
Calculation.create(dataset, formula, name, group)
return self._success('created calculation: %s' % name, dataset_id)
def action(dataset):
if json_file:
calculations = safe_json_loads(json_file.file.read())
Calculation.create_from_list_or_dict(dataset, calculations)
success_message = 'created calculations from JSON'
elif formula is None or name is None:
raise ArgumentError('Must provide both formula and name argume'
'nts, or json_file argument')
else:
Calculation.create(dataset, formula, name, group)
return self._success('created calculation: %s' % name, dataset_id)
def _action(dataset):
if json_file:
calculations = json.loads(json_file.file.read())
Calculation.create_from_list_or_dict(dataset, calculations)
success_message = ('created calculations from JSON for dataset'
':%s' % dataset_id)
elif formula is None or name is None:
raise ArgumentError('Must provide both formula and name argume'
'nts, or json_file argument')
else:
Calculation.create(dataset, formula, name, group)
success_message = 'created calculation: %s for dataset: %s' % (
name, dataset_id)
return {self.SUCCESS: success_message}
def test_save(self):
calculation = self._save_observations_and_calculation()
self.assertTrue(isinstance(calculation, Calculation))
record = calculation.record
self.assertTrue(isinstance(record, dict))
self.assertTrue(Calculation.FORMULA in record.keys())
self.assertTrue(Calculation.STATE in record.keys())
record = Calculation.find(self.dataset)[0].record
self.assertEqual(record[Calculation.STATE], Calculation.STATE_READY)
self.assertTrue(Calculation(record).is_ready)
def test_save_non_existent_group(self):
self._save_observations()
assert_raises(ParseError,
Calculation().save,
self.dataset,
self.formula,
self.name,
group_str='NON_EXISTENT_GROUP')
try:
Calculation().save(self.dataset,
self.formula,
self.name,
group_str='NON_EXISTENT_GROUP')
except ParseError as e:
self.assertTrue('Group' in e.__str__())
def action(dataset):
calculation = Calculation.find_one(dataset.dataset_id, name, group)
if calculation:
calculation.delete(dataset)
message = 'deleted calculation: \'%s\' for dataset: %s' % (
name, dataset.dataset_id)
return {self.SUCCESS: message}
def calculations(self, include_aggs=True, only_aggs=False):
"""Return the calculations for this dataset.
:param include_aggs: Include aggregations, default True.
:param only_aggs: Exclude non-aggregations, default False.
"""
return Calculation.find(self, include_aggs, only_aggs)
def calculations(self, include_aggs=True, only_aggs=False):
"""Return the calculations for this dataset.
:param include_aggs: Include aggregations, default True.
:param only_aggs: Exclude non-aggregations, default False.
"""
return Calculation.find(self, include_aggs, only_aggs)
def action(dataset):
calculation = Calculation.find_one(dataset.dataset_id, name, group)
if calculation:
calculation.delete(dataset)
return {self.SUCCESS: 'deleted calculation: \'%s\'' % name,
Dataset.ID: dataset.dataset_id}
def __wait_for_calculation_ready(self, dataset_id, name):
while True:
calculation = Calculation.find_one(dataset_id, name)
if calculation.is_ready:
break
sleep(self.SLEEP_DELAY)
def action(dataset):
calculation = Calculation.find_one(dataset.dataset_id, name, group)
if calculation:
calculation.delete(dataset)
return self._success('deleted calculation: \'%s\'' % name,
dataset_id)
def action(dataset):
calculation = Calculation.find_one(dataset.dataset_id, name, group)
if calculation:
calculation.delete(dataset)
return self._success('deleted calculation: \'%s\'' % name,
dataset_id)
def test_save(self):
record = self._save_observations_and_calculation()
self.assertTrue(isinstance(record, dict))
self.assertTrue(Calculation.FORMULA in record.keys())
self.assertTrue(Calculation.STATE in record.keys())
record = Calculation.find(self.dataset)[0].record
self.assertEqual(record[Calculation.STATE], Calculation.STATE_READY)
self.assertTrue(Calculation(record).is_ready)
def test_fail_then_create(self):
response = json.loads(self.__post_formula())
self.__verify_create(response)
# Overwrite as failed
calc = Calculation.find_one(self.dataset_id, self.name)
calc.update({calc.STATE: calc.STATE_FAILED})
# Test we can still add a calculation
self.name = 'test2'
response = json.loads(self.__post_formula())
self.__verify_create(response)
def test_create_multiple_with_group(self):
self.dataset_id = self._post_file()
groups = ['risk_factor', 'risk_factor,food_type', 'food_type']
dataset = self.__test_create_from_json(
'good_eats_group.calculations.json', non_agg_cols=2, ex_len=6)
for group in groups:
self.assertTrue(group in dataset.aggregated_datasets_dict.keys())
dframe = dataset.aggregated_dataset(group).dframe()
for column in Calculation().split_groups(group):
self.assertTrue(column in dframe.columns)
def test_fail_in_background(self):
dataset_id = self._post_file('wp_data.csv')
group = 'wp_id'
self._wait_for_dataset_state(dataset_id)
self.controller.create(dataset_id,
'newest(submit_date,functional)',
'wp_functional',
group=group)
self.controller.create(dataset_id,
'max(submit_date)',
'latest_submit_date',
group=group)
# Update the name to cause has pending to be true and infinite retries.
# It will fail after 10 retries.
calc = Calculation.find_one(dataset_id, 'latest_submit_date', group)
calc.update({calc.NAME: 'another_name'})
update = {
'wp_id': 'D',
'functional': 'yes',
}
self.__post_update(dataset_id, update)
update = {
'submit_date': '2013-01-08',
'wp_id': 'A',
'functional': 'no',
}
self.__post_update(dataset_id, update)
while True:
dataset = Dataset.find_one(dataset_id)
calcs_not_pending = [
c.state != c.STATE_PENDING for c in dataset.calculations()]
if not len(dataset.pending_updates) and all(calcs_not_pending):
break
sleep(self.SLEEP_DELAY)
for c in dataset.calculations():
self.assertEqual(c.STATE_FAILED, c.state)
self.assertTrue('Traceback' in c.error_message)
def test_save_set_aggregation_id(self):
record = self._save_observations_and_calculation('max(amount)')
agg_id = self.dataset.aggregated_datasets_dict['']
calculation = Calculation.find(self.dataset)[0]
self.assertEqual(agg_id, calculation.aggregation_id)
def test_save_set_aggregation(self):
record = self._save_observations_and_calculation('max(amount)')
calculation = Calculation.find(self.dataset)[0]
self.assertEqual('max', calculation.aggregation)
def test_sets_dependent_calculations(self):
self._save_observations_and_calculation()
self.name = 'test1'
self._save_calculation('test')
calculation = Calculation.find_one(self.dataset.dataset_id, 'test')
self.assertEqual(calculation.dependent_calculations, ['test1'])
def _save_calculation(self, formula):
if not formula:
formula = self.formula
return Calculation.create(self.dataset, formula, self.name)
def calculations(self):
"""Return the calculations for this dataset."""
return Calculation.find(self)
def test_find(self):
record = self._save_observations_and_calculation()
rows = Calculation.find(self.dataset)
new_record = rows[0].record
status = new_record.pop(Calculation.STATE)
self.assertEqual(status, Calculation.STATE_READY)
def test_find(self):
self._save_observations_and_calculation()
rows = Calculation.find(self.dataset)
new_record = rows[0].record
status = new_record.pop(Calculation.STATE)
self.assertEqual(status, Calculation.STATE_READY)
def _save_calculation(self, formula):
if not formula:
formula = self.formula
return Calculation.create(self.dataset, formula, self.name)
def action(dataset):
result = Calculation.find(dataset)
return [x.clean_record for x in result]
def action(dataset):
result = Calculation.find(dataset)
return [x.clean_record for x in result]
