def validate_hive(self): """Ensure hive partition was created as expected.""" table_name = 'course_blocks' output_dir = url_path_join(self.data_dir, 'output', table_name) for file_name in ('_SUCCESS', 'part-00000', 'part-00001'): actual_output_file = url_path_join(self.warehouse_path, table_name, self.partition, file_name) actual_output_target = get_target_for_local_server(actual_output_file) self.assertTrue(actual_output_target.exists(), '{} not created'.format(file_name)) actual_output = actual_output_target.open('r').read() expected_output_file = url_path_join(output_dir, file_name) expected_output_target = get_target_for_local_server(expected_output_file) expected_output = expected_output_target.open('r').read() self.assertEqual(actual_output, expected_output)
def validate_hive(self): """Ensure hive partition was created.""" hourly_partition = self.DATE.strftime(self.HOURLY_PARTITION_FORMAT) hive_partition = url_path_join(self.warehouse_path, "problem_response_location", "dt=" + hourly_partition) partition_target = get_target_for_local_server(hive_partition) self.assertTrue(partition_target.exists())
def validate_hive(self): """Ensure hive partition was created.""" hourly_partition = self.DATE.strftime(self.HOURLY_PARTITION_FORMAT) hive_partition = url_path_join(self.warehouse_path, "problem_response_location", "dt=" + hourly_partition) partition_target = get_target_for_local_server(hive_partition) self.assertTrue(partition_target.exists())
def validate_reports(self): """Check the generated reports against the expected output files.""" actual_output_targets = self.get_targets_from_remote_path(self.report_output_root) self.assertEqual(len(actual_output_targets), 2) for course_id in ('OpenCraft_PRDemo1_2016', 'OpenCraft_PRDemo2_2016'): report_file_name = '{}_problem_response.csv'.format(course_id) actual_output_targets = self.get_targets_from_remote_path(self.report_output_root, "*{}".format(report_file_name)) self.assertEqual(len(actual_output_targets), 1, '{} not created in {}'.format(report_file_name, self.report_output_root)) actual_output = actual_output_targets[0].open('r').read() expected_output_file = os.path.join(self.data_dir, 'output', 'problem_response', report_file_name) expected_output = get_target_for_local_server(expected_output_file).open('r').read() self.assertEqual(actual_output, expected_output)
def validate_reports(self): """Check the generated reports against the expected output files.""" actual_output_targets = self.get_targets_from_remote_path( self.report_output_root) self.assertEqual(len(actual_output_targets), 2) for course_id in ('OpenCraft_PRDemo1_2016', 'OpenCraft_PRDemo2_2016'): report_file_name = '{}_problem_response.csv'.format(course_id) actual_output_targets = self.get_targets_from_remote_path( self.report_output_root, "*{}".format(report_file_name)) self.assertEqual( len(actual_output_targets), 1, '{} not created in {}'.format(report_file_name, self.report_output_root)) actual_output = actual_output_targets[0].open('r').read() expected_output_file = os.path.join(self.data_dir, 'output', 'problem_response', report_file_name) expected_output = get_target_for_local_server( expected_output_file).open('r').read() self.assertEqual(actual_output, expected_output)
def validate_marker(self, marker_path): """Ensure marker file was created.""" marker_file = url_path_join(marker_path, '_SUCCESS') marker_target = get_target_for_local_server(marker_file) self.assertTrue(marker_target.exists())
def validate_marker(self, marker_path): """Ensure marker file was created.""" marker_file = url_path_join(marker_path, '_SUCCESS') marker_target = get_target_for_local_server(marker_file) self.assertTrue(marker_target.exists())