def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT): if priority > ImportResource.PRIORITY_DEFAULT: filter_limit = settings.SIS_IMPORT_LIMIT['group']['high'] else: filter_limit = settings.SIS_IMPORT_LIMIT['group']['default'] course_ids = super(GroupManager, self).get_queryset().filter( priority=priority, queue_id__isnull=True ).order_by( 'provisioned_date' ).values_list('course_id', flat=True).distinct()[:filter_limit] if not len(course_ids): raise EmptyQueueException() imp = Import(priority=priority, csv_type='group') imp.save() # Mark the groups as in process, and reset the priority super(GroupManager, self).get_queryset().filter( course_id__in=list(course_ids) ).update( priority=ImportResource.PRIORITY_DEFAULT, queue_id=imp.pk ) return imp
def priority_user_import(sender, **kwargs): user = kwargs['instance'] if user.priority == user.PRIORITY_IMMEDIATE: post_save.disconnect(priority_user_import, sender=User) try: imp = Import(priority=user.priority, csv_type='user') imp.save() user.priority = user.PRIORITY_DEFAULT user.queue_id = imp.pk user.save() try: imp.csv_path = UserBuilder([user]).build() except Exception: imp.csv_errors = traceback.format_exc() if imp.csv_path: imp.import_csv() else: user.queue_id = None user.priority = user.PRIORITY_HIGH user.save() imp.delete() except Exception as err: log.error('Immediate user provision failed: {}'.format(err)) user.priority = user.PRIORITY_HIGH user.save() post_save.connect(priority_user_import, sender=User)
def test_type_name(self): imp = Import() self.assertEquals(imp.type_name, None) imp = Import(csv_type='fake') self.assertEquals(imp.type_name, 'fake') imp = Import(csv_type='course') self.assertEquals(imp.type_name, 'Course')
def priority_user_import(sender, **kwargs): user = kwargs['instance'] if user.priority == PRIORITY_IMMEDIATE: post_save.disconnect(priority_user_import, sender=User) try: imp = Import(priority=user.priority, csv_type='user') imp.save() user.priority = PRIORITY_DEFAULT user.queue_id = imp.pk user.save() try: imp.csv_path = UserBuilder([user]).build() except: imp.csv_errors = traceback.format_exc() if imp.csv_path: imp.import_csv() else: user.queue_id = None user.priority = PRIORITY_HIGH user.save() imp.delete() except Exception, err: log.error('Immediate user provision failed: %s' % (err)) user.priority = PRIORITY_HIGH user.save() post_save.connect(priority_user_import, sender=User)
def test_is_completed(self): imp = Import() self.assertEquals(imp.is_completed(), False) imp = Import(post_status=200, canvas_progress=10) self.assertEquals(imp.is_completed(), False) imp = Import(post_status=200, canvas_progress=100) self.assertEquals(imp.is_completed(), True)
def handle(self, *args, **options): if not len(args): raise CommandError("Usage: create_workshop_courses <path> " "<workshop_name><term_sis_id>") file_path = args[0] workshop_name = args[1] term_sis_id = args[2] account_sis_id = 'course-request-sandbox' with open(file_path, 'r') as infile: file_data = infile.read() netids = file_data.splitlines() csvdata = Collector() for netid in netids: try: person = get_person_by_netid(netid.strip()) except UserPolicyException as err: print "Skipped user %s: %s" % (netid, err) continue if not csvdata.add(UserCSV(person)): continue course_sis_id = '%s-%s-%s' % ( term_sis_id, re.sub(r'[^\w]', '-', workshop_name.lower()), person.uwnetid) short_name = '%s %s' % (workshop_name, date.today().year) long_name = '%s Sandbox' % short_name csvdata.add(CourseCSV( course_id=course_sis_id, short_name=short_name, long_name=long_name, account_id=account_sis_id, term_id=term_sis_id, status='active')) csvdata.add(EnrollmentCSV( course_id=course_sis_id, person=person, role=Enrollment.INSTRUCTOR_ROLE, status='active')) csv_path = csvdata.write() if csv_path: imp = Import(priority=PRIORITY_DEFAULT, csv_type='course', csv_path=csv_path) imp.save() imp.import_csv()
def queue_all(self): pks = super(AdminManager, self).get_queryset().filter( queue_id__isnull=True, account__is_deleted__isnull=True).values_list('pk', flat=True) if not len(pks): raise EmptyQueueException() imp = Import(priority=Admin.PRIORITY_DEFAULT, csv_type='admin') imp.save() super(AdminManager, self).get_queryset().update(queue_id=imp.pk) return imp
def test_dequeue(self, mock_update): dt = datetime.now() r = Group.objects.dequeue(Import(pk=1, priority=Group.PRIORITY_HIGH, canvas_state='imported', post_status=200, canvas_progress=100, monitor_date=dt)) mock_update.assert_called_with( priority=Group.PRIORITY_DEFAULT, queue_id=None, provisioned_date=dt) r = Group.objects.dequeue(Import(pk=1, priority=Group.PRIORITY_HIGH)) mock_update.assert_called_with(queue_id=None)
def test_process_warnings(self): imp = Import() empty_warning = [] one_warning = [['users.csv', 'oops']] two_warnings = [ ['users.csv', 'oops'], [ 'courses.csv', '2013-spring-MSIS-550-B--, Course is not a valid course' ] ] self.assertEqual(imp._process_warnings(empty_warning), empty_warning) self.assertEqual(imp._process_warnings(one_warning), one_warning) self.assertEqual(imp._process_warnings(two_warnings), one_warning)
def queue_unused_courses(self, term_id): try: term = Term.objects.get(term_id=term_id) if (term.deleted_unused_courses_date is not None or term.queue_id is not None): raise EmptyQueueException() except Term.DoesNotExist: term = Term(term_id=term_id) term.save() imp = Import(priority=Term.PRIORITY_DEFAULT, csv_type='unused_course') imp.save() term.queue_id = imp.pk term.save() return imp
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT): filter_limit = settings.SIS_IMPORT_LIMIT['enrollment']['default'] pks = super(EnrollmentManager, self).get_queryset().filter( priority=priority, queue_id__isnull=True).order_by('last_modified').values_list( 'pk', flat=True)[:filter_limit] if not len(pks): raise EmptyQueueException() imp = Import(priority=priority, csv_type='enrollment') imp.save() super(EnrollmentManager, self).get_queryset().filter( pk__in=list(pks)).update(queue_id=imp.pk) return imp
def test_dequeue(self, mock_update): dt = datetime.now() r = Term.objects.dequeue( Import(pk=1, priority=Term.PRIORITY_HIGH, canvas_state='imported', post_status=200, canvas_progress=100, monitor_date=dt)) mock_update.assert_called_with(queue_id=None, deleted_unused_courses_date=dt)
def handle(self, *args, **options): imp = Import(csv_type="account") try: imp.csv_path = AccountBuilder().build() except Exception: imp.csv_errors = traceback.format_exc() imp.save() if imp.csv_path: imp.import_csv() self.update_job()
def test_dependent_model(self): imp = Import() self.assertRaises(ImportError, imp.dependent_model) imp = Import(csv_type='fake') self.assertRaises(ImportError, imp.dependent_model) for csv_type in ['enrollment', 'user', 'course', 'admin', 'group']: imp = Import(csv_type=csv_type) self.assertTrue(issubclass(imp.dependent_model(), ImportResource))
def test_dequeue_imported(self, mock_purge, mock_filter): dt = datetime.now() r = Enrollment.objects.dequeue( Import(pk=1, priority=Enrollment.PRIORITY_HIGH, canvas_state='imported', post_status=200, canvas_progress=100, monitor_date=dt)) mock_filter.assert_called_with(queue_id=1, priority__gt=Enrollment.PRIORITY_NONE)
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT): if priority > User.PRIORITY_DEFAULT: filter_limit = settings.SIS_IMPORT_LIMIT['user']['high'] else: filter_limit = settings.SIS_IMPORT_LIMIT['user']['default'] pks = super(UserManager, self).get_queryset().filter( priority=priority, queue_id__isnull=True).order_by( 'provisioned_date', 'added_date').values_list('pk', flat=True)[:filter_limit] if not len(pks): raise EmptyQueueException() imp = Import(csv_type='user', priority=priority) if priority == User.PRIORITY_HIGH: imp.override_sis_stickiness = True imp.save() super(UserManager, self).get_queryset().filter( pk__in=list(pks)).update(queue_id=imp.pk) return imp
def queue_by_priority(self, priority=ImportResource.PRIORITY_DEFAULT): if priority > Course.PRIORITY_DEFAULT: filter_limit = settings.SIS_IMPORT_LIMIT['course']['high'] else: filter_limit = settings.SIS_IMPORT_LIMIT['course']['default'] pks = super(CourseManager, self).get_queryset().filter( priority=priority, course_type=Course.SDB_TYPE, queue_id__isnull=True, provisioned_error__isnull=True).order_by( 'provisioned_date', 'added_date').values_list('pk', flat=True)[:filter_limit] if not len(pks): raise EmptyQueueException() imp = Import(priority=priority, csv_type='course') imp.save() super(CourseManager, self).get_queryset().filter( pk__in=list(pks)).update(queue_id=imp.pk) return imp
def handle(self, *args, **options): file_path = options.get('file_path') workshop_name = options.get('workshop_name') term_sis_id = options.get('term_sis_id') account_sis_id = options.get('account_sis_id') with open(file_path, 'r') as infile: file_data = infile.read() netids = file_data.splitlines() csvdata = Collector() for netid in netids: try: person = get_person_by_netid(netid.strip()) except UserPolicyException as err: print("Skipped user '{}': {}".format(netid, err)) continue if not csvdata.add(UserCSV(person)): continue course_sis_id = '-'.join([ term_sis_id, re.sub(r'[^\w]', '-', workshop_name.lower()), person.uwnetid ]) short_name = '{} {}'.format(date.today().year, workshop_name) long_name = '{} Sandbox'.format(short_name) csvdata.add( CourseCSV(course_id=course_sis_id, short_name=short_name, long_name=long_name, account_id=account_sis_id, term_id=term_sis_id, status='active')) csvdata.add( EnrollmentCSV(course_id=course_sis_id, person=person, role=get_instructor_sis_import_role(), status=ENROLLMENT_ACTIVE)) csv_path = csvdata.write_files() if csv_path: imp = Import(priority=Course.PRIORITY_DEFAULT, csv_type='course', csv_path=csv_path) imp.save() imp.import_csv()
def test_dequeue_not_imported(self, mock_update): r = Enrollment.objects.dequeue( Import(pk=1, priority=Enrollment.PRIORITY_HIGH)) mock_update.assert_called_with(queue_id=None)
def test_is_cleanly_imported(self): imp = Import() self.assertEquals(imp.is_cleanly_imported(), False) imp = Import(post_status=200, canvas_progress=100) self.assertEquals(imp.is_cleanly_imported(), False) imp = Import(post_status=200, canvas_progress=100, canvas_state='imported_with_messages', canvas_warnings='[["users.csv", "oops"]]') self.assertEquals(imp.is_cleanly_imported(), False) imp = Import(post_status=200, canvas_progress=100, canvas_state='imported_with_messages') self.assertEquals(imp.is_cleanly_imported(), True) imp = Import(post_status=200, canvas_progress=100, canvas_state='imported') self.assertEquals(imp.is_cleanly_imported(), True)
def test_is_imported(self): imp = Import() self.assertEquals(imp.is_imported(), False) imp = Import(post_status=200, canvas_progress=100) self.assertEquals(imp.is_imported(), False) imp = Import(post_status=200, canvas_progress=100, canvas_state='imported_with_messages') self.assertEquals(imp.is_imported(), True) imp = Import(post_status=200, canvas_progress=100, canvas_state='imported') self.assertEquals(imp.is_imported(), True)