def test_join_when_invalid_grade_negative_only_course_of_student(
spark_session):
student_data = [(1, 'Mario', 'CS'), (2, 'Paula', 'Biology'),
(3, 'Juan', 'Business')]
course_data = [(1, 4, 'CS'), (2, 2, 'Biology'), (3, 4, 'Business')]
grade_data = [(1, 1, -10), (2, 2, 95), (3, 3, 84)]
students_df = spark_session.createDataFrame(
student_data, ['id', 'name', 'student_major'])
courses_df = spark_session.createDataFrame(
course_data, ['id', 'credits', 'course_major'])
grades_df = spark_session.createDataFrame(
grade_data, ['student_id', 'course_id', 'grade'])
expected_data = [
(3, 'Juan', 3, 4, 84, 'Business'),
(2, 'Paula', 2, 2, 95, 'Biology'),
]
expected_df = spark_session.createDataFrame(expected_data, [
'student_id', 'name', 'course_id', 'credits', 'grade', 'student_major'
])
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
actual_df = top_students_finder.join_student_data()
assert expected_df.collect() == actual_df.collect()
def test_join_when_more_than_one_course_per_student(spark_session):
student_data = [(1, 'Mario', 'CS'), (2, 'Paula', 'Biology'),
(3, 'Juan', 'Business')]
course_data = [(1, 4, 'CS'), (2, 2, 'Biology'), (3, 4, 'Business'),
(4, 6, 'CS'), (5, 8, 'CS')]
grade_data = [(1, 1, 90), (2, 2, 95), (3, 3, 84), (1, 4, 65), (1, 5, 87)]
students_df = spark_session.createDataFrame(
student_data, ['id', 'name', 'student_major'])
courses_df = spark_session.createDataFrame(
course_data, ['id', 'credits', 'course_major'])
grades_df = spark_session.createDataFrame(
grade_data, ['student_id', 'course_id', 'grade'])
expected_data = [
(3, 'Juan', 3, 4, 84, 'Business'),
(1, 'Mario', 4, 6, 65, 'CS'),
(1, 'Mario', 5, 8, 87, 'CS'),
(1, 'Mario', 1, 4, 90, 'CS'),
(2, 'Paula', 2, 2, 95, 'Biology'),
]
expected_df = spark_session.createDataFrame(expected_data, [
'student_id', 'name', 'course_id', 'credits', 'grade', 'student_major'
])
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
actual_df = top_students_finder.join_student_data()
assert expected_df.collect() == actual_df.collect()
def main():
spark = SparkSession.builder.appName('database').master(
'local').getOrCreate()
students_df, courses_df, grades_df = load_data(spark)
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
top_three_students = top_students_finder.get_top_n_students(2)
top_three_students.show()
def test_join_when_students_null(spark_session):
course_data = [(1, 4, 'CS'), (2, 2, 'Biology'), (3, 4, 'Business')]
grade_data = [(1, 1, 90), (2, 2, 95), (3, 3, 84)]
students_df = None
courses_df = spark_session.createDataFrame(
course_data, ['id', 'credits', 'course_major'])
grades_df = spark_session.createDataFrame(
grade_data, ['student_id', 'course_id', 'grade'])
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
actual_df = top_students_finder.join_student_data()
assert actual_df is None
def test_join_when_grades_null(spark_session):
student_data = [(1, 'Mario', 'CS'), (2, 'Paula', 'Biology'),
(3, 'Juan', 'Business')]
course_data = [(1, 4, 'CS'), (2, 2, 'Biology'), (3, 4, 'Business')]
students_df = spark_session.createDataFrame(
student_data, ['id', 'name', 'student_major'])
courses_df = spark_session.createDataFrame(
course_data, ['id', 'credits', 'course_major'])
grades_df = None
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
actual_df = top_students_finder.join_student_data()
assert actual_df is None
def test_join_when_no_students(spark_session):
student_data = []
course_data = [(1, 4, 'CS'), (2, 2, 'Biology'), (3, 4, 'Business'),
(4, 6, 'CS'), (5, 8, 'CS')]
grade_data = [(1, 1, 90), (2, 2, 95), (3, 3, 84), (1, 4, 65), (1, 5, 87)]
field = [
StructField('id', IntegerType(), True),
StructField('name', IntegerType(), True),
StructField('student_major', StringType(), True)
]
students_df = spark_session.createDataFrame(student_data,
schema=StructType(field))
courses_df = spark_session.createDataFrame(
course_data, ['id', 'credits', 'course_major'])
grades_df = spark_session.createDataFrame(
grade_data, ['student_id', 'course_id', 'grade'])
top_students_finder = TopStudentsFinder(students_df, courses_df, grades_df)
actual_df = top_students_finder.join_student_data()
assert actual_df.rdd.isEmpty()