def handle(self, *args, **options):
term = Term.objects.all()[0]
terms = get_all_terms(term.key)
departments = Department.objects.all().values_list('code', flat=True)
for t in terms:
for department in departments:
existing = set(Section.objects.values_list('code', flat=True))
active = set()
try:
courses = simplejson.load(urllib.urlopen(COURSES_URL % (t, department)))
if courses:
for course in courses:
code = course['CourseCode']
active.add(code)
if code in existing:
# update section
section_object = Section.objects.get(code=code)
self.stdout.write('updating section "%s"\n' % code)
try:
section_object.course.departments.add(Department.objects.get(code=department))
except Department.DoesNotExist:
continue
self.refresh_one_section(section_object, course)
else:
# add new course and section
code_slug = slugify(code).upper()
course_code = code[:-3]
course_code_slug = code_slug[:-3]
course_number = ''.join([s for s in course_code if s.isdigit()])
if not course_number:
self.stdout.write('unknown number for course "%s" - deleting...\n' % course_code)
continue
else:
course_number = int(course_number)
course_object, added = Course.objects.get_or_create(code=course_code,
code_slug=course_code_slug,
number=course_number)
if added: self.stdout.write('added course "%s"\n' % course_code)
try:
course_object.departments.add(Department.objects.get(code=department))
except Department.DoesNotExist:
continue
object = Section(term=term, code=code, code_slug=code_slug)
object.course = course_object
self.stdout.write('adding section "%s"\n' % code)
self.refresh_one_section(object, course)
except simplejson.scanner.JSONDecodeError:
self.stdout.write('error accessing "%s"' % (COURSES_URL % (t, department)))
areas = RequirementArea.objects.all().values_list('code', flat=True)
for area in areas:
try:
area_object = RequirementArea.objects.get(code=area)
if urllib.urlopen(COURSES_URL % (t, area)).read():
courses = simplejson.load(urllib.urlopen(COURSES_URL % (t, area)))
for course in courses:
code = course['CourseCode']
try:
object = Section.objects.get(code=code)
object.course.requirement_areas.add(area_object)
except Section.DoesNotExist:
self.stdout.write('unknown section "%s"' % code)
continue
except RequirementArea.DoesNotExist:
continue
# Remove courses that have been deleted from the catalog (or whose codes
# have changed... can't tell the difference)
stale = existing - active
if stale:
Course.objects.filter(code__in=stale).delete()
def handle(self, *args, **options):
term = Term.objects.all()[0]
terms = get_all_terms(term.key)
departments = Department.objects.all().values_list('code', flat=True)
existing = set(Section.objects.filter(term=term).values_list('code', flat=True))
active = set()
for t in terms:
for department in departments:
try:
sections = simplejson.load(urllib.urlopen(COURSES_URL % (t, department)))
if sections:
for section in sections:
# Ensure that the existing variable contains sections added in previous iterations of the import
# TODO: Make this more efficient (i.e. memoize the data so we don't have to reassign every loop)
existing = set(Section.objects.filter(term=term).values_list('code', flat=True))
try:
section_code = section['CourseCode']
except TypeError:
# Sometimes `section` holds the string "Message" for some reason
continue
active.add(section_code)
if section_code in existing:
# update section
try:
section_object = Section.objects.get(code=section_code, term=term)
except MultipleObjectsReturned:
self.stdout.write('multiple sections of "%s" for term "%s"' % (section_code, term))
continue
self.stdout.write('updating section "%s"\n' % section_code)
try:
section_object.course.departments.add(Department.objects.get(code=department))
except Department.DoesNotExist:
continue
self.refresh_one_section(section_object, section)
else:
# add new section and possible course
section_code_slug = slugify(section_code).upper()
course_code = section_code[:-3]
course_code_slug = section_code_slug[:-3]
course_number = ''.join([s for s in course_code if s.isdigit()])
if not course_number:
self.stdout.write('unknown number for course "%s" - deleting...\n' % course_code)
continue
else:
course_number = int(course_number)
course_object, added = Course.objects.get_or_create(code=course_code,
code_slug=course_code_slug,
number=course_number)
if added: self.stdout.write('added course "%s"\n' % course_code)
try:
course_object.departments.add(Department.objects.get(code=department))
except Department.DoesNotExist:
continue
section_object = Section(term=term, code=section_code, code_slug=section_code_slug)
section_object.course = course_object
self.stdout.write('adding section "%s"\n' % section_code)
self.refresh_one_section(section_object, section)
except simplejson.scanner.JSONDecodeError:
self.stdout.write('error accessing "%s"' % (COURSES_URL % (t, department)))
areas = RequirementArea.objects.all().values_list('code', flat=True)
for area in areas:
try:
area_object = RequirementArea.objects.get(code=area)
if urllib.urlopen(COURSES_URL % (t, area)).read():
sections = simplejson.load(urllib.urlopen(COURSES_URL % (t, area)))
for section in sections:
try:
code = section['CourseCode']
except TypeError:
# Sometimes section holds the string "Message" for some reason
continue
try:
section_object = Section.objects.get(code=code, term=term)
section_object.course.requirement_areas.add(area_object)
except MultipleObjectsReturned:
# FIXME: This should never happen at this stage! Multiple section errors should be caught
# in the previous `for section in sections:` block above, otherwise some sections will not
# have the area requirement data properly set.
self.stdout.write('multiple sections of "%s" for term "%s"' % (section_code, term))
continue
except Section.DoesNotExist:
self.stdout.write('unknown section "%s"' % code)
continue
except RequirementArea.DoesNotExist:
continue
# Remove courses that have been deleted from the catalog (or whose codes
# have changed... can't tell the difference)
stale = existing - active
if stale:
Section.objects.filter(code__in=stale).delete()
# Add a timestamp of refresh time (i.e. a new RefreshHistory record)
new_history = RefreshHistory(
last_refresh_date=datetime.now(),
term=term,
type=RefreshHistory.FULL
)
new_history.save()