def validate_hive(self):
"""Ensure hive partition was created as expected."""
table_name = 'course_blocks'
output_dir = url_path_join(self.data_dir, 'output', table_name)
for file_name in ('_SUCCESS', 'part-00000', 'part-00001'):
actual_output_file = url_path_join(self.warehouse_path, table_name, self.partition, file_name)
actual_output_target = get_target_for_local_server(actual_output_file)
self.assertTrue(actual_output_target.exists(), '{} not created'.format(file_name))
actual_output = actual_output_target.open('r').read()
expected_output_file = url_path_join(output_dir, file_name)
expected_output_target = get_target_for_local_server(expected_output_file)
expected_output = expected_output_target.open('r').read()
self.assertEqual(actual_output, expected_output)
def validate_hive(self):
"""Ensure hive partition was created."""
hourly_partition = self.DATE.strftime(self.HOURLY_PARTITION_FORMAT)
hive_partition = url_path_join(self.warehouse_path, "problem_response_location",
"dt=" + hourly_partition)
partition_target = get_target_for_local_server(hive_partition)
self.assertTrue(partition_target.exists())
def validate_hive(self):
"""Ensure hive partition was created."""
hourly_partition = self.DATE.strftime(self.HOURLY_PARTITION_FORMAT)
hive_partition = url_path_join(self.warehouse_path,
"problem_response_location",
"dt=" + hourly_partition)
partition_target = get_target_for_local_server(hive_partition)
self.assertTrue(partition_target.exists())
def validate_reports(self):
"""Check the generated reports against the expected output files."""
actual_output_targets = self.get_targets_from_remote_path(self.report_output_root)
self.assertEqual(len(actual_output_targets), 2)
for course_id in ('OpenCraft_PRDemo1_2016', 'OpenCraft_PRDemo2_2016'):
report_file_name = '{}_problem_response.csv'.format(course_id)
actual_output_targets = self.get_targets_from_remote_path(self.report_output_root, "*{}".format(report_file_name))
self.assertEqual(len(actual_output_targets), 1, '{} not created in {}'.format(report_file_name, self.report_output_root))
actual_output = actual_output_targets[0].open('r').read()
expected_output_file = os.path.join(self.data_dir, 'output', 'problem_response', report_file_name)
expected_output = get_target_for_local_server(expected_output_file).open('r').read()
self.assertEqual(actual_output, expected_output)
def validate_reports(self):
"""Check the generated reports against the expected output files."""
actual_output_targets = self.get_targets_from_remote_path(
self.report_output_root)
self.assertEqual(len(actual_output_targets), 2)
for course_id in ('OpenCraft_PRDemo1_2016', 'OpenCraft_PRDemo2_2016'):
report_file_name = '{}_problem_response.csv'.format(course_id)
actual_output_targets = self.get_targets_from_remote_path(
self.report_output_root, "*{}".format(report_file_name))
self.assertEqual(
len(actual_output_targets), 1,
'{} not created in {}'.format(report_file_name,
self.report_output_root))
actual_output = actual_output_targets[0].open('r').read()
expected_output_file = os.path.join(self.data_dir, 'output',
'problem_response',
report_file_name)
expected_output = get_target_for_local_server(
expected_output_file).open('r').read()
self.assertEqual(actual_output, expected_output)
def validate_marker(self, marker_path):
"""Ensure marker file was created."""
marker_file = url_path_join(marker_path, '_SUCCESS')
marker_target = get_target_for_local_server(marker_file)
self.assertTrue(marker_target.exists())
def validate_marker(self, marker_path):
"""Ensure marker file was created."""
marker_file = url_path_join(marker_path, '_SUCCESS')
marker_target = get_target_for_local_server(marker_file)
self.assertTrue(marker_target.exists())