def sync(self, get_files=None, get_catalog=None, force=False):
if get_files is None:
get_files = self.get_files
if get_catalog is None:
from rpi_courses import CourseCatalog
get_catalog = CourseCatalog.from_string
self.forced = force
for filename in get_files(self.latest_semester):
semester = self.semesters.get(filename)
# if latest semester or newer semester
if (not semester) or semester == self.latest_semester:
try:
with closing(urllib2.urlopen(filename)) as page:
logger.debug("OPEN " + filename)
if force or (semester and semester.date_updated is not None):
last_mod = dateutil.parser.parse(dict(page.info())['last-modified']).replace(tzinfo=None)
if not force and last_mod <= semester.date_updated:
logger.debug("Skipping b/c of mod date: %r <= %r" % (last_mod, semester.date_updated))
continue
catalog = get_catalog(page.read(), filename)
except urllib2.URLError:
logger.debug("Failed to fetch url (%r)" % (filename))
continue
if not force and self.latest_semester and semester == self.latest_semester:
continue # already up-to-date
logger.debug("Semester: %r => %s-%s" % (
filename,
catalog.year,
catalog.month,
))
semester_obj, created = Semester.objects.get_or_create(
year=catalog.year,
month=catalog.month,
ref=filename,
defaults={
'visible': False,
'name': catalog.name,
})
self.create_courses(catalog, semester_obj)
# catalog doesn't support this for now.
#self.create_crosslistings(semester_obj, set(catalog.crosslistings.values()))
semester_obj.save() # => update date_updated property
if created:
logger.debug(' CREATE SEMESTER ' + repr(semester_obj))
self.notifier.requires_notification()
else:
logger.debug(' EXISTS SEMESTER ' + repr(semester_obj))
if self.sections_changed:
sections_modified.send(sender=self, semester=semester_obj)
self.clear_unused(semester_obj)
def sync(self, get_files=None, get_catalog=None):
"Performs the updating of the database data from RPI's SIS"
if get_files is None:
from rpi_courses import list_rocs_xml_files
get_files = list_rocs_xml_files
if get_catalog is None:
from rpi_courses import ROCSCourseCatalog
get_catalog = ROCSCourseCatalog.from_url
for filename in get_files():
name = self.FILE_RE.finditer(filename).next().groups()[0]
semester = self.semesters.get(name + '.xml')
# if latest semester or newer semester
if (not semester) or semester == self.latest_semester:
catalog = get_catalog(filename)
if self.latest_semester and semester == self.latest_semester and catalog.datetime <= self.latest_semester.date_updated:
continue # already up-to-date
logger.debug('found catalog for: %r %r' %
(catalog.year, catalog.month))
semester_obj = None
try:
semester_obj, created = Semester.objects.get_or_create(
year=catalog.year,
month=catalog.month,
defaults={
'visible': True,
'name': catalog.name,
'ref': filename,
})
except IntegrityError as error:
logger.debug(' DUPLICATE SEMESTER ' + repr(semester_obj) +
': ' + repr(error))
continue
self.create_courses(catalog, semester_obj)
self.create_crosslistings(semester_obj,
set(catalog.crosslistings.values()))
semester_obj.save() # => update date_updated property
if created:
logger.debug(' CREATE SEMESTER ' + repr(semester_obj))
self.notifier.requires_notification()
else:
logger.debug(' EXISTS SEMESTER ' + repr(semester_obj))
if self.sections_changed:
sections_modified.send(sender=self, semester=semester_obj)
self.clear_unused(semester_obj)
def sync(self, get_files=None, get_catalog=None):
"Performs the updating of the database data from RPI's SIS"
if get_files is None:
from rpi_courses import list_rocs_xml_files
get_files = list_rocs_xml_files
if get_catalog is None:
from rpi_courses import ROCSCourseCatalog
get_catalog = ROCSCourseCatalog.from_url
for filename in get_files():
name = self.FILE_RE.finditer(filename).next().groups()[0]
semester = self.semesters.get(name + '.xml')
# if latest semester or newer semester
if (not semester) or semester == self.latest_semester:
catalog = get_catalog(filename)
if self.latest_semester and semester == self.latest_semester and catalog.datetime <= self.latest_semester.date_updated:
continue # already up-to-date
logger.debug('found catalog for: %r %r' % (catalog.year, catalog.month))
semester_obj = None
try:
semester_obj, created = Semester.objects.get_or_create(
year=catalog.year,
month=catalog.month,
defaults={
'visible': True,
'name': catalog.name,
'ref': filename,
})
except IntegrityError as error:
logger.debug(' DUPLICATE SEMESTER ' + repr(semester_obj) + ': ' + repr(error))
continue
self.create_courses(catalog, semester_obj)
self.create_crosslistings(semester_obj, set(catalog.crosslistings.values()))
semester_obj.save() # => update date_updated property
if created:
logger.debug(' CREATE SEMESTER ' + repr(semester_obj))
self.notifier.requires_notification()
else:
logger.debug(' EXISTS SEMESTER ' + repr(semester_obj))
if self.sections_changed:
sections_modified.send(sender=self, semester=semester_obj)
self.clear_unused(semester_obj)
def sync(self, get_files=None, get_catalog=None, force=False):
if get_files is None:
get_files = self.get_files
if get_catalog is None:
from rpi_courses import CourseCatalog
get_catalog = CourseCatalog.from_string
self.forced = force
for filename in get_files(self.latest_semester):
semester = self.semesters.get(filename)
# if latest semester or newer semester
if (not semester) or semester == self.latest_semester:
try:
with closing(urllib2.urlopen(filename)) as page:
logger.debug("OPEN " + filename)
if force or (semester
and semester.date_updated is not None):
last_mod = dateutil.parser.parse(
dict(page.info())['last-modified']).replace(
tzinfo=None)
if not force and last_mod <= semester.date_updated:
logger.debug(
"Skipping b/c of mod date: %r <= %r" %
(last_mod, semester.date_updated))
continue
catalog = get_catalog(page.read(), filename)
except urllib2.URLError:
logger.debug("Failed to fetch url (%r)" % (filename))
continue
if not force and self.latest_semester and semester == self.latest_semester:
continue # already up-to-date
logger.debug("Semester: %r => %s-%s" % (
filename,
catalog.year,
catalog.month,
))
semester_obj = None
try:
semester_obj, created = Semester.objects.get_or_create(
ref=filename,
defaults={
'year': catalog.year,
'month': catalog.month,
'visible': False,
'name': catalog.name,
})
except IntegrityError as error:
logger.debug(' DUPLICATE SEMESTER ' + repr(semester_obj) +
': ' + repr(error))
continue
self.create_courses(catalog, semester_obj)
# catalog doesn't support this for now.
#self.create_crosslistings(semester_obj, set(catalog.crosslistings.values()))
semester_obj.save() # => update date_updated property
if created:
logger.debug(' CREATE SEMESTER ' + repr(semester_obj))
self.notifier.requires_notification()
else:
logger.debug(' EXISTS SEMESTER ' + repr(semester_obj))
if self.sections_changed:
sections_modified.send(sender=self, semester=semester_obj)
self.clear_unused(semester_obj)