def get_json_value(value): """Parse JSON value based on type.""" if is_float_nan(value): value = JSON_NULL elif isinstance(value, np.int64): value = int(value) elif isinstance(value, np.bool_): value = bool(value) return value
def _is_potential_date(value): return not (is_float_nan(value) or isinstance(value, bool))
def test_update_after_agg_group(self): dataset_id = self._post_file('wp_data.csv') group = 'wp_id' self._wait_for_dataset_state(dataset_id) test_calculations = { 'newest(submit_date,functional)': 'wp_functional', 'max(submit_date)': 'latest_submit_date', 'ratio(functional in ["yes"], 1)': 'wp_func_ratio'} expected_results = {'wp_id': ['A', 'B', 'C', 'n/a'], 'wp_functional': ['yes', 'no', 'yes', 'yes'], 'wp_func_ratio': [1.0, 0.0, 1.0, 1.0], 'wp_func_ratio_denominator': [1, 1, 1, 1], 'wp_func_ratio_numerator': [1.0, 0.0, 1.0, 1.0], 'latest_submit_date': [1356998400, 1357084800, 1357171200, 1357257600]} expected_results_after = { 'wp_id': ['A', 'B', 'C', 'D', 'n/a'], 'wp_functional': ['no', 'no', 'yes', 'yes'], 'wp_func_ratio': [0.5, 0.0, 1.0, 1.0, 1.0], 'wp_func_ratio_denominator': [2.0, 1.0, 1.0, 1.0, 1.0], 'wp_func_ratio_numerator': [1.0, 0.0, 1.0, 1.0, 1.0], 'latest_submit_date': [1357603200.0, 1357084800.0, 1357171200.0, 1357257600.0]} for formula, name in test_calculations.items(): results = json.loads(self.controller.create( dataset_id, formula, name, group=group)) self.assertTrue(self.controller.SUCCESS in results) dataset = Dataset.find_one(dataset_id) previous_num_rows = dataset.num_rows while True: dataset = Dataset.find_one(dataset_id) if dataset.aggregated_dataset(group) and all( [not c.is_pending for c in dataset.calculations()]): break sleep(self.SLEEP_DELAY) agg_dframe = dataset.aggregated_dataset(group).dframe() self.assertEqual(set(expected_results.keys()), set(agg_dframe.columns.tolist())) for column, results in expected_results.items(): self.assertEqual(results, agg_dframe[column].tolist()) update = { 'wp_id': 'D', 'functional': 'yes', } self.__post_update(dataset_id, update) update = { 'submit_date': '2013-01-08', 'wp_id': 'A', 'functional': 'no', } self.__post_update(dataset_id, update) while True: dataset = Dataset.find_one(dataset_id) current_num_rows = dataset.num_rows if not len(dataset.pending_updates): break sleep(self.SLEEP_DELAY) dataset = Dataset.find_one(dataset_id) agg_dframe = dataset.aggregated_dataset(group).dframe() self.assertEqual(current_num_rows, previous_num_rows + 2) self.assertEqual(set(expected_results_after.keys()), set(agg_dframe.columns.tolist())) for column, results in expected_results_after.items(): column = [x for x in agg_dframe[column].tolist() if not is_float_nan(x)] self.assertEqual(results, column)