def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "hoid:q", ["help","output=","input_json_data=","dept=","term=","level="])
        except getopt.error, msg:
            raise Usage(msg)

        verbose= True
        output = os.path.dirname(os.path.realpath(__file__)) + '/../data/course_data.json'
        json_data = None
        dept = None
        level = None
        term = None

        # option processing
        for option, value in opts:
            if option == "-q":
                verbose = False
            if option in ("-h", "--help"):
                raise Usage(help_message)
            if option in ("-i", "--input_json_data"):
                json_data = value
            if option in ("-o", "--output"):
                output = value
            if option in ("-d", "--dept"):
                dept = value
            if option in ("-t", "--term"):
                term = value
            if option in ("-l", "--level"):
                level = value
                
        c = testudo.crawler(term=term, verbose=verbose)
        
        if json_data:
            # Load exising JSON data (faster)
            courses = json.load(open(json_data, 'rb'))
        else:
            # Fetch course data from server
            if dept:                
#                 courses = c.get_courses(dept=dept, level=level)      
                courses = []
                courses.extend(c.get_courses(dept=dept, level='UGRAD'))
                courses.extend(c.get_courses(dept=dept, level='GRAD'))          
            else:
                courses = []
                for d in dept_list:
                    courses.extend(c.get_courses(dept=d, level='UGRAD'))
                    courses.extend(c.get_courses(dept=d, level='GRAD'))                                       
            
            # Error checking, with counting the number of each field
            for attribute, value in count_fields(courses).iteritems():                
                if value<2:                                      
                    print "Field " + attribute + " has a small number of fields, " + str(value) + ". Report this problem directly to Administrators."
                    return 0
            
        json.dump(courses, open(output, 'wb'), indent=2)
        print "1"
        return "[Python] Crawling Successful.\n"
def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "hoid:q", ["help","output=","input_json_data=","dept="])
        except getopt.error, msg:
            raise Usage(msg)

        verbose= True
        output = 'data/course_data.json'
        json_data = None
        dept = None

        # option processing
        for option, value in opts:
            if option == "-q":
                verbose = False
            if option in ("-h", "--help"):
                raise Usage(help_message)
            if option in ("-i", "--input_json_data"):
                json_data = value
            if option in ("-o", "--output"):
                output = value
            if option in ("-d", "--dept"):
                dept = value

        c = testudo.crawler(term='201101', verbose=verbose)
        if json_data:
            # Load exising JSON data (faster)
            courses = json.load(open(json_data, 'rb'))
        else:
            # Fetch course data from server
            if dept:
                courses = c.get_courses(dept=dept)
            else:
                courses = c.get_all_courses()
            json.dump(courses, open(output, 'wb'), indent=2)

        if csv:
            course_writer = csv.writer(open('data/courses.csv', 'wb'), delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
            section_writer = csv.writer(open('data/sections.csv', 'wb'), delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
            class_time_writer = csv.writer(open('data/class_times.csv', 'wb'), delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)

            for c in courses:
                if c['sections']:
                    for s in c['sections']:
                        if s['class_times']:
                            for ct in s['class_times']:
                                class_time_writer.writerow([c['code'], s['section']] + ct.values())
                        del s['class_times']
                        section_writer.writerow([c['code']] + s.values())
                del c['sections']
                course_writer.writerow(c.values())
 def setUp(self):
     self.crawler = testudo.crawler(term='201101', verbose=True)
     pass