def _test_calculator(self): self.dframe = self.dataset.dframe() columns = self.dframe.columns.tolist() self.start_num_cols = len(columns) self.added_num_cols = 0 column_labels_to_slugs = { column_attrs[Dataset.LABEL]: (column_name) for (column_name, column_attrs) in self.dataset.schema.items() } self.label_list, self.slugified_key_list = [ list(ary) for ary in zip(*column_labels_to_slugs.items()) ] for idx, formula in enumerate(self.calculations): name = 'test-%s' % idx self.parser.validate_formula(formula) calculator = Calculator(self.dataset) calculation = Calculation() calculation.save(self.dataset, formula, name, self.group) calculator.calculate_columns([calculation]) self.column_labels_to_slugs = self.dataset.schema.labels_to_slugs self._test_calculation_results(name, formula)
def _test_calculator(self): self.dframe = self.dataset.dframe() columns = self.dframe.columns.tolist() self.start_num_cols = len(columns) self.added_num_cols = 0 column_labels_to_slugs = { column_attrs[Dataset.LABEL]: (column_name) for (column_name, column_attrs) in self.dataset.schema.items() } self.label_list, self.slugified_key_list = [ list(ary) for ary in zip(*column_labels_to_slugs.items()) ] for idx, formula in enumerate(self.calculations): name = 'test-%s' % idx Parser.validate_formula(formula, self.dataset) calculation = Calculation() calculation.save(self.dataset, formula, name, self.group) self.now = now() calculate_columns(self.dataset, [calculation]) self.column_labels_to_slugs = self.dataset.schema.labels_to_slugs self._test_calculation_results(name, formula)
def test_disallow_delete_dependent_calculation(self): record = self._save_observations_and_calculation() self.name = 'test1' record = self._save_calculation('test') calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, ['test1']) calculation = Calculation.find_one(self.dataset.dataset_id, 'test') assert_raises(DependencyError, calculation.delete, self.dataset)
def test_save_improper_formula_no_data(self): assert_raises(ParseError, Calculation().save, self.dataset, 'NON_EXISTENT_COLUMN', self.name) try: Calculation().save(self.dataset, 'NON_EXISTENT_COLUMN', self.name) except ParseError as e: self.assertTrue('No schema' in e.__str__())
def test_disallow_delete_dependent_calculation(self): self._save_observations_and_calculation() self.name = 'test1' self._save_calculation('test') calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, ['test1']) calculation = Calculation.find_one(self.dataset.dataset_id, 'test') assert_raises(DependencyError, calculation.delete, self.dataset)
def test_save_unparsable_formula_no_data(self): assert_raises(ParseError, Calculation().save, self.dataset, '=NON_EXISTENT_COLUMN', self.name) try: Calculation().save(self.dataset, '=NON_EXISTENT_COLUMN', self.name) except ParseError as e: self.assertTrue('Parse Failure' in e.__str__())
def test_removes_dependent_calculations(self): record = self._save_observations_and_calculation() self.name = 'test1' record = self._save_calculation('test') calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, ['test1']) calculation = Calculation.find_one(self.dataset.dataset_id, 'test1') calculation.delete(self.dataset) calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, [])
def test_removes_dependent_calculations(self): self._save_observations_and_calculation() self.name = 'test1' self._save_calculation('test') calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, ['test1']) calculation = Calculation.find_one(self.dataset.dataset_id, 'test1') calculation.delete(self.dataset) calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, [])
def action(dataset): if json_file: calculations = safe_json_loads(json_file.file.read()) Calculation.create_from_list_or_dict(dataset, calculations) success_message = "created calculations from JSON for dataset" ":%s" % dataset_id elif formula is None or name is None: raise ArgumentError("Must provide both formula and name argume" "nts, or json_file argument") else: Calculation.create(dataset, formula, name, group) success_message = "created calculation: %s for dataset: %s" % (name, dataset_id) return {self.SUCCESS: success_message}
def action(dataset): if json_file: calculations = safe_json_loads(json_file.file.read()) Calculation.create_from_list_or_dict(dataset, calculations) success_message = 'created calculations from JSON' elif formula is None or name is None: raise ArgumentError('Must provide both formula and name argume' 'nts, or json_file argument') else: Calculation.create(dataset, formula, name, group) return self._success('created calculation: %s' % name, dataset_id)
def _action(dataset): if json_file: calculations = json.loads(json_file.file.read()) Calculation.create_from_list_or_dict(dataset, calculations) success_message = ('created calculations from JSON for dataset' ':%s' % dataset_id) elif formula is None or name is None: raise ArgumentError('Must provide both formula and name argume' 'nts, or json_file argument') else: Calculation.create(dataset, formula, name, group) success_message = 'created calculation: %s for dataset: %s' % ( name, dataset_id) return {self.SUCCESS: success_message}
def test_save(self): calculation = self._save_observations_and_calculation() self.assertTrue(isinstance(calculation, Calculation)) record = calculation.record self.assertTrue(isinstance(record, dict)) self.assertTrue(Calculation.FORMULA in record.keys()) self.assertTrue(Calculation.STATE in record.keys()) record = Calculation.find(self.dataset)[0].record self.assertEqual(record[Calculation.STATE], Calculation.STATE_READY) self.assertTrue(Calculation(record).is_ready)
def test_save_non_existent_group(self): self._save_observations() assert_raises(ParseError, Calculation().save, self.dataset, self.formula, self.name, group_str='NON_EXISTENT_GROUP') try: Calculation().save(self.dataset, self.formula, self.name, group_str='NON_EXISTENT_GROUP') except ParseError as e: self.assertTrue('Group' in e.__str__())
def action(dataset): calculation = Calculation.find_one(dataset.dataset_id, name, group) if calculation: calculation.delete(dataset) message = 'deleted calculation: \'%s\' for dataset: %s' % ( name, dataset.dataset_id) return {self.SUCCESS: message}
def calculations(self, include_aggs=True, only_aggs=False): """Return the calculations for this dataset. :param include_aggs: Include aggregations, default True. :param only_aggs: Exclude non-aggregations, default False. """ return Calculation.find(self, include_aggs, only_aggs)
def action(dataset): calculation = Calculation.find_one(dataset.dataset_id, name, group) if calculation: calculation.delete(dataset) return {self.SUCCESS: 'deleted calculation: \'%s\'' % name, Dataset.ID: dataset.dataset_id}
def __wait_for_calculation_ready(self, dataset_id, name): while True: calculation = Calculation.find_one(dataset_id, name) if calculation.is_ready: break sleep(self.SLEEP_DELAY)
def action(dataset): calculation = Calculation.find_one(dataset.dataset_id, name, group) if calculation: calculation.delete(dataset) return self._success('deleted calculation: \'%s\'' % name, dataset_id)
def test_save(self): record = self._save_observations_and_calculation() self.assertTrue(isinstance(record, dict)) self.assertTrue(Calculation.FORMULA in record.keys()) self.assertTrue(Calculation.STATE in record.keys()) record = Calculation.find(self.dataset)[0].record self.assertEqual(record[Calculation.STATE], Calculation.STATE_READY) self.assertTrue(Calculation(record).is_ready)
def test_fail_then_create(self): response = json.loads(self.__post_formula()) self.__verify_create(response) # Overwrite as failed calc = Calculation.find_one(self.dataset_id, self.name) calc.update({calc.STATE: calc.STATE_FAILED}) # Test we can still add a calculation self.name = 'test2' response = json.loads(self.__post_formula()) self.__verify_create(response)
def test_create_multiple_with_group(self): self.dataset_id = self._post_file() groups = ['risk_factor', 'risk_factor,food_type', 'food_type'] dataset = self.__test_create_from_json( 'good_eats_group.calculations.json', non_agg_cols=2, ex_len=6) for group in groups: self.assertTrue(group in dataset.aggregated_datasets_dict.keys()) dframe = dataset.aggregated_dataset(group).dframe() for column in Calculation().split_groups(group): self.assertTrue(column in dframe.columns)
def test_fail_in_background(self): dataset_id = self._post_file('wp_data.csv') group = 'wp_id' self._wait_for_dataset_state(dataset_id) self.controller.create(dataset_id, 'newest(submit_date,functional)', 'wp_functional', group=group) self.controller.create(dataset_id, 'max(submit_date)', 'latest_submit_date', group=group) # Update the name to cause has pending to be true and infinite retries. # It will fail after 10 retries. calc = Calculation.find_one(dataset_id, 'latest_submit_date', group) calc.update({calc.NAME: 'another_name'}) update = { 'wp_id': 'D', 'functional': 'yes', } self.__post_update(dataset_id, update) update = { 'submit_date': '2013-01-08', 'wp_id': 'A', 'functional': 'no', } self.__post_update(dataset_id, update) while True: dataset = Dataset.find_one(dataset_id) calcs_not_pending = [ c.state != c.STATE_PENDING for c in dataset.calculations()] if not len(dataset.pending_updates) and all(calcs_not_pending): break sleep(self.SLEEP_DELAY) for c in dataset.calculations(): self.assertEqual(c.STATE_FAILED, c.state) self.assertTrue('Traceback' in c.error_message)
def test_save_set_aggregation_id(self): record = self._save_observations_and_calculation('max(amount)') agg_id = self.dataset.aggregated_datasets_dict[''] calculation = Calculation.find(self.dataset)[0] self.assertEqual(agg_id, calculation.aggregation_id)
def test_save_set_aggregation(self): record = self._save_observations_and_calculation('max(amount)') calculation = Calculation.find(self.dataset)[0] self.assertEqual('max', calculation.aggregation)
def test_sets_dependent_calculations(self): self._save_observations_and_calculation() self.name = 'test1' self._save_calculation('test') calculation = Calculation.find_one(self.dataset.dataset_id, 'test') self.assertEqual(calculation.dependent_calculations, ['test1'])
def _save_calculation(self, formula): if not formula: formula = self.formula return Calculation.create(self.dataset, formula, self.name)
def calculations(self): """Return the calculations for this dataset.""" return Calculation.find(self)
def test_find(self): record = self._save_observations_and_calculation() rows = Calculation.find(self.dataset) new_record = rows[0].record status = new_record.pop(Calculation.STATE) self.assertEqual(status, Calculation.STATE_READY)
def test_find(self): self._save_observations_and_calculation() rows = Calculation.find(self.dataset) new_record = rows[0].record status = new_record.pop(Calculation.STATE) self.assertEqual(status, Calculation.STATE_READY)
def action(dataset): result = Calculation.find(dataset) return [x.clean_record for x in result]