def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT):
if priority > ImportResource.PRIORITY_DEFAULT:
filter_limit = settings.SIS_IMPORT_LIMIT['group']['high']
else:
filter_limit = settings.SIS_IMPORT_LIMIT['group']['default']
course_ids = super(GroupManager, self).get_queryset().filter(
priority=priority, queue_id__isnull=True
).order_by(
'provisioned_date'
).values_list('course_id', flat=True).distinct()[:filter_limit]
if not len(course_ids):
raise EmptyQueueException()
imp = Import(priority=priority, csv_type='group')
imp.save()
# Mark the groups as in process, and reset the priority
super(GroupManager, self).get_queryset().filter(
course_id__in=list(course_ids)
).update(
priority=ImportResource.PRIORITY_DEFAULT, queue_id=imp.pk
)
return imp
def priority_user_import(sender, **kwargs):
user = kwargs['instance']
if user.priority == user.PRIORITY_IMMEDIATE:
post_save.disconnect(priority_user_import, sender=User)
try:
imp = Import(priority=user.priority, csv_type='user')
imp.save()
user.priority = user.PRIORITY_DEFAULT
user.queue_id = imp.pk
user.save()
try:
imp.csv_path = UserBuilder([user]).build()
except Exception:
imp.csv_errors = traceback.format_exc()
if imp.csv_path:
imp.import_csv()
else:
user.queue_id = None
user.priority = user.PRIORITY_HIGH
user.save()
imp.delete()
except Exception as err:
log.error('Immediate user provision failed: {}'.format(err))
user.priority = user.PRIORITY_HIGH
user.save()
post_save.connect(priority_user_import, sender=User)
def test_type_name(self):
imp = Import()
self.assertEquals(imp.type_name, None)
imp = Import(csv_type='fake')
self.assertEquals(imp.type_name, 'fake')
imp = Import(csv_type='course')
self.assertEquals(imp.type_name, 'Course')
def priority_user_import(sender, **kwargs):
user = kwargs['instance']
if user.priority == PRIORITY_IMMEDIATE:
post_save.disconnect(priority_user_import, sender=User)
try:
imp = Import(priority=user.priority, csv_type='user')
imp.save()
user.priority = PRIORITY_DEFAULT
user.queue_id = imp.pk
user.save()
try:
imp.csv_path = UserBuilder([user]).build()
except:
imp.csv_errors = traceback.format_exc()
if imp.csv_path:
imp.import_csv()
else:
user.queue_id = None
user.priority = PRIORITY_HIGH
user.save()
imp.delete()
except Exception, err:
log.error('Immediate user provision failed: %s' % (err))
user.priority = PRIORITY_HIGH
user.save()
post_save.connect(priority_user_import, sender=User)
def test_is_completed(self):
imp = Import()
self.assertEquals(imp.is_completed(), False)
imp = Import(post_status=200, canvas_progress=10)
self.assertEquals(imp.is_completed(), False)
imp = Import(post_status=200, canvas_progress=100)
self.assertEquals(imp.is_completed(), True)
def handle(self, *args, **options):
if not len(args):
raise CommandError("Usage: create_workshop_courses <path> "
"<workshop_name><term_sis_id>")
file_path = args[0]
workshop_name = args[1]
term_sis_id = args[2]
account_sis_id = 'course-request-sandbox'
with open(file_path, 'r') as infile:
file_data = infile.read()
netids = file_data.splitlines()
csvdata = Collector()
for netid in netids:
try:
person = get_person_by_netid(netid.strip())
except UserPolicyException as err:
print "Skipped user %s: %s" % (netid, err)
continue
if not csvdata.add(UserCSV(person)):
continue
course_sis_id = '%s-%s-%s' % (
term_sis_id,
re.sub(r'[^\w]', '-', workshop_name.lower()),
person.uwnetid)
short_name = '%s %s' % (workshop_name, date.today().year)
long_name = '%s Sandbox' % short_name
csvdata.add(CourseCSV(
course_id=course_sis_id, short_name=short_name,
long_name=long_name, account_id=account_sis_id,
term_id=term_sis_id, status='active'))
csvdata.add(EnrollmentCSV(
course_id=course_sis_id, person=person,
role=Enrollment.INSTRUCTOR_ROLE, status='active'))
csv_path = csvdata.write()
if csv_path:
imp = Import(priority=PRIORITY_DEFAULT, csv_type='course',
csv_path=csv_path)
imp.save()
imp.import_csv()
def queue_all(self):
pks = super(AdminManager, self).get_queryset().filter(
queue_id__isnull=True,
account__is_deleted__isnull=True).values_list('pk', flat=True)
if not len(pks):
raise EmptyQueueException()
imp = Import(priority=Admin.PRIORITY_DEFAULT, csv_type='admin')
imp.save()
super(AdminManager, self).get_queryset().update(queue_id=imp.pk)
return imp
def test_dequeue(self, mock_update):
dt = datetime.now()
r = Group.objects.dequeue(Import(pk=1,
priority=Group.PRIORITY_HIGH,
canvas_state='imported',
post_status=200,
canvas_progress=100,
monitor_date=dt))
mock_update.assert_called_with(
priority=Group.PRIORITY_DEFAULT, queue_id=None,
provisioned_date=dt)
r = Group.objects.dequeue(Import(pk=1, priority=Group.PRIORITY_HIGH))
mock_update.assert_called_with(queue_id=None)
def test_process_warnings(self):
imp = Import()
empty_warning = []
one_warning = [['users.csv', 'oops']]
two_warnings = [
['users.csv', 'oops'],
[
'courses.csv',
'2013-spring-MSIS-550-B--, Course is not a valid course'
]
]
self.assertEqual(imp._process_warnings(empty_warning), empty_warning)
self.assertEqual(imp._process_warnings(one_warning), one_warning)
self.assertEqual(imp._process_warnings(two_warnings), one_warning)
def queue_unused_courses(self, term_id):
try:
term = Term.objects.get(term_id=term_id)
if (term.deleted_unused_courses_date is not None
or term.queue_id is not None):
raise EmptyQueueException()
except Term.DoesNotExist:
term = Term(term_id=term_id)
term.save()
imp = Import(priority=Term.PRIORITY_DEFAULT, csv_type='unused_course')
imp.save()
term.queue_id = imp.pk
term.save()
return imp
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT):
filter_limit = settings.SIS_IMPORT_LIMIT['enrollment']['default']
pks = super(EnrollmentManager, self).get_queryset().filter(
priority=priority,
queue_id__isnull=True).order_by('last_modified').values_list(
'pk', flat=True)[:filter_limit]
if not len(pks):
raise EmptyQueueException()
imp = Import(priority=priority, csv_type='enrollment')
imp.save()
super(EnrollmentManager, self).get_queryset().filter(
pk__in=list(pks)).update(queue_id=imp.pk)
return imp
def test_dequeue(self, mock_update):
dt = datetime.now()
r = Term.objects.dequeue(
Import(pk=1,
priority=Term.PRIORITY_HIGH,
canvas_state='imported',
post_status=200,
canvas_progress=100,
monitor_date=dt))
mock_update.assert_called_with(queue_id=None,
deleted_unused_courses_date=dt)
def handle(self, *args, **options):
imp = Import(csv_type="account")
try:
imp.csv_path = AccountBuilder().build()
except Exception:
imp.csv_errors = traceback.format_exc()
imp.save()
if imp.csv_path:
imp.import_csv()
self.update_job()
def test_dependent_model(self):
imp = Import()
self.assertRaises(ImportError, imp.dependent_model)
imp = Import(csv_type='fake')
self.assertRaises(ImportError, imp.dependent_model)
for csv_type in ['enrollment', 'user', 'course', 'admin', 'group']:
imp = Import(csv_type=csv_type)
self.assertTrue(issubclass(imp.dependent_model(), ImportResource))
def test_dequeue_imported(self, mock_purge, mock_filter):
dt = datetime.now()
r = Enrollment.objects.dequeue(
Import(pk=1,
priority=Enrollment.PRIORITY_HIGH,
canvas_state='imported',
post_status=200,
canvas_progress=100,
monitor_date=dt))
mock_filter.assert_called_with(queue_id=1,
priority__gt=Enrollment.PRIORITY_NONE)
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT):
if priority > User.PRIORITY_DEFAULT:
filter_limit = settings.SIS_IMPORT_LIMIT['user']['high']
else:
filter_limit = settings.SIS_IMPORT_LIMIT['user']['default']
pks = super(UserManager, self).get_queryset().filter(
priority=priority, queue_id__isnull=True).order_by(
'provisioned_date',
'added_date').values_list('pk', flat=True)[:filter_limit]
if not len(pks):
raise EmptyQueueException()
imp = Import(csv_type='user', priority=priority)
if priority == User.PRIORITY_HIGH:
imp.override_sis_stickiness = True
imp.save()
super(UserManager, self).get_queryset().filter(
pk__in=list(pks)).update(queue_id=imp.pk)
return imp
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT):
if priority > Course.PRIORITY_DEFAULT:
filter_limit = settings.SIS_IMPORT_LIMIT['course']['high']
else:
filter_limit = settings.SIS_IMPORT_LIMIT['course']['default']
pks = super(CourseManager, self).get_queryset().filter(
priority=priority,
course_type=Course.SDB_TYPE,
queue_id__isnull=True,
provisioned_error__isnull=True).order_by(
'provisioned_date',
'added_date').values_list('pk', flat=True)[:filter_limit]
if not len(pks):
raise EmptyQueueException()
imp = Import(priority=priority, csv_type='course')
imp.save()
super(CourseManager, self).get_queryset().filter(
pk__in=list(pks)).update(queue_id=imp.pk)
return imp
def handle(self, *args, **options):
file_path = options.get('file_path')
workshop_name = options.get('workshop_name')
term_sis_id = options.get('term_sis_id')
account_sis_id = options.get('account_sis_id')
with open(file_path, 'r') as infile:
file_data = infile.read()
netids = file_data.splitlines()
csvdata = Collector()
for netid in netids:
try:
person = get_person_by_netid(netid.strip())
except UserPolicyException as err:
print("Skipped user '{}': {}".format(netid, err))
continue
if not csvdata.add(UserCSV(person)):
continue
course_sis_id = '-'.join([
term_sis_id,
re.sub(r'[^\w]', '-', workshop_name.lower()), person.uwnetid
])
short_name = '{} {}'.format(date.today().year, workshop_name)
long_name = '{} Sandbox'.format(short_name)
csvdata.add(
CourseCSV(course_id=course_sis_id,
short_name=short_name,
long_name=long_name,
account_id=account_sis_id,
term_id=term_sis_id,
status='active'))
csvdata.add(
EnrollmentCSV(course_id=course_sis_id,
person=person,
role=get_instructor_sis_import_role(),
status=ENROLLMENT_ACTIVE))
csv_path = csvdata.write_files()
if csv_path:
imp = Import(priority=Course.PRIORITY_DEFAULT,
csv_type='course',
csv_path=csv_path)
imp.save()
imp.import_csv()
def test_dequeue_not_imported(self, mock_update):
r = Enrollment.objects.dequeue(
Import(pk=1, priority=Enrollment.PRIORITY_HIGH))
mock_update.assert_called_with(queue_id=None)
def test_is_cleanly_imported(self):
imp = Import()
self.assertEquals(imp.is_cleanly_imported(), False)
imp = Import(post_status=200, canvas_progress=100)
self.assertEquals(imp.is_cleanly_imported(), False)
imp = Import(post_status=200,
canvas_progress=100,
canvas_state='imported_with_messages',
canvas_warnings='[["users.csv", "oops"]]')
self.assertEquals(imp.is_cleanly_imported(), False)
imp = Import(post_status=200,
canvas_progress=100,
canvas_state='imported_with_messages')
self.assertEquals(imp.is_cleanly_imported(), True)
imp = Import(post_status=200,
canvas_progress=100,
canvas_state='imported')
self.assertEquals(imp.is_cleanly_imported(), True)
def test_is_imported(self):
imp = Import()
self.assertEquals(imp.is_imported(), False)
imp = Import(post_status=200, canvas_progress=100)
self.assertEquals(imp.is_imported(), False)
imp = Import(post_status=200,
canvas_progress=100,
canvas_state='imported_with_messages')
self.assertEquals(imp.is_imported(), True)
imp = Import(post_status=200,
canvas_progress=100,
canvas_state='imported')
self.assertEquals(imp.is_imported(), True)
