Exemplo n.º 1
0
def locateHeadings(headingsList,sheetAddress):

    '''Generate the column numbers holding the headings
    '''
    headingsLocation=[]
    for heading in headingsList:
        headingsLocation.append(findCol(sheetAddress,heading))

    return headingsLocation
Exemplo n.º 2
0
def getCourseInfo(neededHeadings, currentSheet):
    """Returns a course object, which contains the necessary info for a course record
        in the database, for the course table

        Will return the subject code (e.g. ANAT), catalog number (e.g. 215) and term (e.g. 2135)
    """

    headingsLocation = []

    for i in range(len(neededHeadings)):
        headingsLocation.append(dbFormat.findCol(currentSheet, neededHeadings[i]))

    courseInfo = Course()  # initialize the course object

    values = []
    for dataCol in headingsLocation:
        values.append(currentSheet.cell(2, dataCol).value)

    courseInfo.subject = values[0]
    courseInfo.catalog = values[1]
    courseInfo.term = values[2]

    return courseInfo
Exemplo n.º 3
0
def main(rawDataLocation):  # pass in the raw data directory
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Open Database
    """ Initialize all the connections to SQLite
        Configure database stuff
    """

    cdLocation = os.getcwd()
    dbName = "\\enrolldata.db"
    dbLocation = cdLocation + dbName

    # try:
    #     os.remove(dbLocation) #"""@@@@@@@@@@ REMOVES THE DB EACH TIME THIS RUNS, FOR TESTING PURPOSES"""
    # except WindowsError:        #WindowsError appreas when there is no database file
    #     pass

    conn = sqlite3.connect(dbLocation)

    c = conn.cursor()

    c.execute("PRAGMA foreign_keys=ON;")  # enables foreign keys

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Open Excel workbooks
    """Initialize the Excel workbook connections for reading using xlrd module
        Find all xls files and then store the location and addresses of all of them
    """
    excelExtension = "xls"
    filesList = findFiles(rawDataLocation, excelExtension)

    wbData = []
    sheetAddress = []
    for i in range(len(filesList)):
        wbData.append(xlrd.open_workbook(filesList[i]))
        sheetAddress.append(
            wbData[i].sheet_by_index(0)
        )  # will access and store the location of the FIRST sheet in each workbook

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Preprocess the excel columns from text to float
    """Use Excel Macros and control it through coms with win32com.client
    """
    preprocessHeadings = [
        "Student ID",
        "Proj Level",
        "Term",
    ]  # these are the headings of columns that are floats and need to be made into floats

    macroLocation = (
        cdLocation + "\\excelMacro\\intToFloat.xlsm"
    )  # """@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ change to make it dynamic"""

    xl = win32com.client.Dispatch("Excel.Application")  # accesses the excel application
    xl.Visible = 1

    wbPre = xl.Workbooks.Open(macroLocation)  # open the workbook containing the macro

    wbList = []

    for i in range(len(filesList)):

        wbList.append(0)

        preprocessHeadingsLocation = dbFormat.findHeadingsList(preprocessHeadings, sheetAddress[i])
        # this only uses one set of table headings so if the table heading columns are different, there will be   problem

        for j in range(len(preprocessHeadingsLocation)):

            preprocessHeadingsLocation[j] = (
                preprocessHeadingsLocation[j] + 1
            )  # the table headings must be incremented by 1 so the macro below can process the correct heading

            xl.Application.Run("makeTexttoFloat", preprocessHeadingsLocation[j], filesList[i])

    wbPre.Save()  # close and save the excel workbook with macros

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Reopens the files using XLRD again (after being updated using macro)
    wbData = []
    sheetAddress = []
    for i in range(len(filesList)):
        wbData.append(xlrd.open_workbook(filesList[i]))
        sheetAddress.append(
            wbData[i].sheet_by_index(0)
        )  # will access and store the location of the FIRST sheet in each workbook

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Create Table Headings

    # students table
    # determining headings from excel and making into table headings for sql [hardcoded but will make into dropdown input for UI]

    studentTableHeadings = ["Student ID", "Program", "Proj Level", "Plan"]
    studentDataTypes = [
        "INTEGER NOT NULL UNIQUE",
        "TEXT NOT NULL",
        "INTEGER NOT NULL",
        "TEXT NOT NULL",
    ]  # where student_id is a unique field
    maxCourses = (
        10
    )  # number of courses that can be enrolled in (should be 5 for A&S for one term, 10 for a year)<<<LOOK INTO IT

    if checkTableExist(c, "students") is False:

        studentSQLHeadings = dbFormat.generateHeading(sheetAddress[0], studentTableHeadings, studentDataTypes)

        studentSQLHeadings = (
            studentSQLHeadings + ", plan2 TEXT, plan3 TEXT"
        )  # amend the headings to add the second plan column (for double majors)

        c.execute("CREATE TABLE students(stud_id INTEGER PRIMARY KEY," + studentSQLHeadings + ")")

        for i in range(maxCourses):
            c.execute("ALTER TABLE students ADD COLUMN course" + str(i + 1) + " INTEGER REFERENCES courses(course_id);")

    # course table
    courseTableHeadings = [
        "Subject",
        "Catalog Number",
        "TERM",
    ]  # catalog number must be null because of suffix letters e.g. PHYG 214A
    courseDataTypes = ["TEXT NOT NULL", "TEXT NOT NULL", "INTEGER NOT NULL"]

    if checkTableExist(c, "courses") is False:

        headingsLocation = []
        for i in range(len(courseTableHeadings)):
            headingsLocation.append(
                dbFormat.findCol(sheetAddress[0], courseTableHeadings[i])
            )  # headingsLocation are found in order of courseTableHeadings, not actual order on sheet

        courseSQLHeadings = dbFormat.generateHeading(sheetAddress[0], courseTableHeadings, courseDataTypes)

        c.execute("CREATE TABLE courses(course_id INTEGER PRIMARY KEY," + courseSQLHeadings + ")")

    # time stamp table
    if checkTableExist(c, "timeRecord") is False:

        c.execute(
            """CREATE TABLE timeRecord 
                    (
                        time_id INTEGER PRIMARY KEY, 
                        timeStam TEXT
                    );"""
        )

        c.execute("""INSERT INTO timeRecord (timeStam) VALUES (?);""", (dateTimeOutput.pythonTime(),))

    # info stored about programs, used to calculate tuition and grants
    if checkTableExist(c, "program_info") is False:

        c.execute(
            """CREATE TABLE program_info 
                    (
                        program_id INTEGER PRIMARY KEY, 
                        program_name TEXT UNIQUE, 
                        unit_fees FLOAT, 
                        formula_fees FLOAT, 
                        program_weight FLOAT,
                        normal_units INTEGER
                    );"""
        )

    # storing values like BIU
    if checkTableExist(c, "constants") is False:

        c.execute(
            """CREATE TABLE constants 
                    (
                        id INTEGER PRIMARY KEY, 
                        name TEXT, 
                        value INTEGER
                    );"""
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    # Pulling student data

    """ Each student list from a workbook is appended to the master student list (studentLists).
        Thus, each item in that array is an array itself, making studentLists a multi-dimensional
        array.
        A tuple in the list courseLists has just the information about the course from a particular
        workbook. It is a one-dimensional array.
        A list in studentList will correspond to a tuple in courseList when the index of the two lists
        are the same. e.g. studentLists[2] will correspond to courseLists[2]
    """

    studentTableHeadings = [
        "Student ID",
        "Program",
        "Proj Level",
        "Plan",
    ]  # restate because some function above messes with it
    courseTableHeadings = ["Subject", "Catalog Number", "Term"]

    studentLists = []
    courseLists = []
    courseNames = []

    for i in range(len(sheetAddress)):
        courseLists.append(getCourseInfo(courseTableHeadings, sheetAddress[i]))
        courseNames.append(courseLists[i].concatenateCourseCode())

    if not dbFormat.checkColumnExist(c, "course_code", "courses"):
        c.execute(
            "ALTER TABLE courses ADD course_code TEXT;"
        )  # adding the column with the concatenateCourseCoded course code

    for i in range(len(sheetAddress)):

        c.execute(
            """SELECT DISTINCT course_id FROM courses WHERE course_code = ? and term = ?""",
            (str(courseNames[i]), str(courseLists[i].term)),
        )
        checkRecordExist = (
            c.fetchall()
        )  # if the record already exists in the course table, it will return the index of the record

        if (
            checkRecordExist == []
        ):  # the course will only be inputted into the database if it has unique course_code and term
            c.execute(
                """INSERT INTO courses(subject, catalog_number, course_code, term) 
                        VALUES(?,?,?,?);""",
                (courseLists[i].subject, courseLists[i].catalog, courseNames[i], courseLists[i].term),
            )

        studentLists.append(getStudents(studentTableHeadings, sheetAddress[i]))

        for stud in studentLists[
            i
        ]:  # OR IGNORE will only input the record if it is unique (does not show the error message)
            c.execute(
                """INSERT OR IGNORE INTO students (student_id, program, proj_level, plan, plan2, plan3) 
                        VALUES(?,?,?,?,?,?);""",
                (stud.studID, stud.program, stud.proj_level, stud.plan, stud.plan2, stud.plan3),
            )

        c.execute(
            """SELECT course_id 
                    FROM courses 
                    WHERE course_id=(SELECT MAX(course_ID) 
                                    FROM courses);"""
        )
        # finds the course_id of the most recently entered course list from the courses table

        # convert the tuple to an integer
        temp = c.fetchone()
        courseNum = 0
        count = 0
        for a in reversed(temp):
            courseNum = courseNum + a * 10 ** count
            count = count + 1

        """ Goes through all the students in the current course list and accesses all rows from the SQL database.
         It will add the courses that a student is enrolled in.
        """
        c.execute("SELECT * FROM students")
        val = c.fetchall()

        for stud in studentLists[i]:  # goes through all the students in current course list
            for row in val:  # goes through all rows of the table
                for i in reversed(
                    range(1, maxCourses + 1)
                ):  # accesses the row's course values, course 1 thru maxCourses
                    if (
                        row[len(row) - i] is None and stud.studID == row[1]
                    ):  # must check if the student is enrolled in the current course
                        c.execute(
                            "UPDATE students SET course" + str(maxCourses + 1 - i) + "=? WHERE student_id=?;",
                            (str(courseNum), row[1]),
                        )
                        break

    # Initialize the program_info table by inserting just the program names
    c.execute("SELECT DISTINCT program FROM students;")
    programList = c.fetchall()

    for i in range(len(programList)):
        val = "".join(programList[i])
        try:
            c.execute("INSERT INTO program_info(program_name) VALUES (?);", (val,))
        except sqlite3.IntegrityError:
            pass

    # Update the timestamp
    c.execute(
        """UPDATE timeRecord 
                SET timeStam = ? 
                WHERE time_id = 1;""",
        (dateTimeOutput.pythonTime(),),
    )  # updates the time stamp for the table after the spreadsheets were pulled

    # Add the credits column (blank)
    if not dbFormat.checkColumnExist(c, "credits", "courses"):
        c.execute("ALTER TABLE courses ADD credits FLOAT;")

    conn.commit()
    conn.close()

    # Closing Excel Spreadsheet (close it at the very end so the user knows the program is still running)
    wbPre.Close(True)
    xl.Application.Quit()  # killing the dispatch
    xl.Visible = 0  # need to make it not visible
    del (xl)
Exemplo n.º 4
0
def getStudents(headingsNeeded, currentSheet):
    """Returns a list of student objects, with each object containing 
        student id, program, year, plan (including double and triple majors)
        
        Also will removes duplicates (who result from double majors) when 
        parsing from the excel spreadsheet. 

        A single student will show up twice if he or she is a double major, 
        but the function will pop off the duplicate and append the double major 
        plan to the appropriate student object
    """

    headingsLocation = []

    for i in range(len(headingsNeeded)):
        headingsLocation.append(dbFormat.findCol(currentSheet, headingsNeeded[i]))

    studList = [Student() for i in range(currentSheet.nrows)]  # create the array of students

    for row in range(currentSheet.nrows):
        """parse the data of each excel row into a Student object, which is in the array"""
        values = []
        for dataCol in headingsLocation:
            values.append(currentSheet.cell(row, dataCol).value)

        studList[row].studID = values[0]
        studList[row].program = values[1]
        studList[row].proj_level = values[2]
        studList[row].plan = values[3]
        studList[row].plan2 = ""
        studList[row].plan3 = ""

    studList = studentProcessing.removeBEDProg(studList)  # prevents BED from being the plan

    popRows = []

    popRows.extend(
        studentProcessing.findImproperStudent(studList)
    )  # finds the headings and whitespaces and flags the index
    for i in studentProcessing.findTriple(studList):
        """finds the triples (accounts for con ed, who MAY have B.ED + double major for their plans, 3 rows)"""
        studList[i + 2].plan3 = studList[i].plan
    popRows.extend(studentProcessing.findTriple(studList))  # flags the index for triples

    for i in reversed(popRows):  # when iterating in reverse, the wrong things do not get popped off the list
        studList.pop(i)  # pop off headings/whitespaces and triples

    popRows = []  # resets the list so we can add duplicate indexes

    for i in studentProcessing.findDuplicate(studList):  # finds the duplicates and adds the indices to the popRows list
        studList[i + 1].plan2 = studList[
            i
        ].plan  # before the duplicate is popped, the second plan is added to the data row
    popRows.extend(studentProcessing.findDuplicate(studList))

    for i in reversed(popRows):  # when iterating in reverse, the wrong things do not get popped off the list
        studList.pop(i)
    popRows = []

    for i in range(len(studList)):
        studList[i].convert()  # finalize input data

    return studList  # returns a list of Student objects