def _resolve_output_field(self):
source_field = self.get_source_fields()[0]
if isinstance(source_field, (IntegerField, DecimalField)):
return FloatField()
return super()._resolve_output_field()
class CourseOverview(TimeStampedModel):
"""
Model for storing and caching basic information about a course.
This model contains basic course metadata such as an ID, display name,
image URL, and any other information that would be necessary to display
a course as part of:
user dashboard (enrolled courses)
course catalog (courses to enroll in)
course about (meta data about the course)
"""
class Meta(object):
app_label = 'course_overviews'
# IMPORTANT: Bump this whenever you modify this model and/or add a migration.
VERSION = 6
# Cache entry versioning.
version = IntegerField()
# Course identification
id = CourseKeyField(db_index=True, primary_key=True, max_length=255)
_location = UsageKeyField(max_length=255)
org = TextField(max_length=255, default='outdated_entry')
display_name = TextField(null=True)
display_number_with_default = TextField()
display_org_with_default = TextField()
# Start/end dates
start = DateTimeField(null=True)
end = DateTimeField(null=True)
advertised_start = TextField(null=True)
announcement = DateTimeField(null=True)
# URLs
course_image_url = TextField()
social_sharing_url = TextField(null=True)
end_of_course_survey_url = TextField(null=True)
# Certification data
certificates_display_behavior = TextField(null=True)
certificates_show_before_end = BooleanField(default=False)
cert_html_view_enabled = BooleanField(default=False)
has_any_active_web_certificate = BooleanField(default=False)
cert_name_short = TextField()
cert_name_long = TextField()
certificate_available_date = DateTimeField(default=None, null=True)
# Grading
lowest_passing_grade = DecimalField(max_digits=5, decimal_places=2, null=True)
# Access parameters
days_early_for_beta = FloatField(null=True)
mobile_available = BooleanField(default=False)
visible_to_staff_only = BooleanField(default=False)
_pre_requisite_courses_json = TextField() # JSON representation of list of CourseKey strings
# Enrollment details
enrollment_start = DateTimeField(null=True)
enrollment_end = DateTimeField(null=True)
enrollment_domain = TextField(null=True)
invitation_only = BooleanField(default=False)
max_student_enrollments_allowed = IntegerField(null=True)
# Catalog information
catalog_visibility = TextField(null=True)
short_description = TextField(null=True)
course_video_url = TextField(null=True)
effort = TextField(null=True)
self_paced = BooleanField(default=False)
marketing_url = TextField(null=True)
eligible_for_financial_aid = BooleanField(default=True)
language = TextField(null=True)
@classmethod
def _create_or_update(cls, course):
"""
Creates or updates a CourseOverview object from a CourseDescriptor.
Does not touch the database, simply constructs and returns an overview
from the given course.
Arguments:
course (CourseDescriptor): any course descriptor object
Returns:
CourseOverview: created or updated overview extracted from the given course
"""
from lms.djangoapps.certificates.api import get_active_web_certificate
from openedx.core.lib.courses import course_image_url
# Workaround for a problem discovered in https://openedx.atlassian.net/browse/TNL-2806.
# If the course has a malformed grading policy such that
# course._grading_policy['GRADE_CUTOFFS'] = {}, then
# course.lowest_passing_grade will raise a ValueError.
# Work around this for now by defaulting to None.
try:
lowest_passing_grade = course.lowest_passing_grade
except ValueError:
lowest_passing_grade = None
display_name = course.display_name
start = course.start
end = course.end
max_student_enrollments_allowed = course.max_student_enrollments_allowed
if isinstance(course.id, CCXLocator):
from lms.djangoapps.ccx.utils import get_ccx_from_ccx_locator
ccx = get_ccx_from_ccx_locator(course.id)
display_name = ccx.display_name
start = ccx.start
end = ccx.due
max_student_enrollments_allowed = ccx.max_student_enrollments_allowed
course_overview = cls.objects.filter(id=course.id)
if course_overview.exists():
log.info('Updating course overview for %s.', unicode(course.id))
course_overview = course_overview.first()
else:
log.info('Creating course overview for %s.', unicode(course.id))
course_overview = cls()
course_overview.version = cls.VERSION
course_overview.id = course.id
course_overview._location = course.location
course_overview.org = course.location.org
course_overview.display_name = display_name
course_overview.display_number_with_default = course.display_number_with_default
course_overview.display_org_with_default = course.display_org_with_default
course_overview.start = start
course_overview.end = end
course_overview.advertised_start = course.advertised_start
course_overview.announcement = course.announcement
course_overview.course_image_url = course_image_url(course)
course_overview.social_sharing_url = course.social_sharing_url
course_overview.certificates_display_behavior = course.certificates_display_behavior
course_overview.certificates_show_before_end = course.certificates_show_before_end
course_overview.cert_html_view_enabled = course.cert_html_view_enabled
course_overview.has_any_active_web_certificate = (get_active_web_certificate(course) is not None)
course_overview.cert_name_short = course.cert_name_short
course_overview.cert_name_long = course.cert_name_long
course_overview.certificate_available_date = course.certificate_available_date
course_overview.lowest_passing_grade = lowest_passing_grade
course_overview.end_of_course_survey_url = course.end_of_course_survey_url
course_overview.days_early_for_beta = course.days_early_for_beta
course_overview.mobile_available = course.mobile_available
course_overview.visible_to_staff_only = course.visible_to_staff_only
course_overview._pre_requisite_courses_json = json.dumps(course.pre_requisite_courses)
course_overview.enrollment_start = course.enrollment_start
course_overview.enrollment_end = course.enrollment_end
course_overview.enrollment_domain = course.enrollment_domain
course_overview.invitation_only = course.invitation_only
course_overview.max_student_enrollments_allowed = max_student_enrollments_allowed
course_overview.catalog_visibility = course.catalog_visibility
course_overview.short_description = CourseDetails.fetch_about_attribute(course.id, 'short_description')
course_overview.effort = CourseDetails.fetch_about_attribute(course.id, 'effort')
course_overview.course_video_url = CourseDetails.fetch_video_url(course.id)
course_overview.self_paced = course.self_paced
if not CatalogIntegration.is_enabled():
course_overview.language = course.language
return course_overview
@classmethod
def load_from_module_store(cls, course_id):
"""
Load a CourseDescriptor, create or update a CourseOverview from it, cache the
overview, and return it.
Arguments:
course_id (CourseKey): the ID of the course overview to be loaded.
Returns:
CourseOverview: overview of the requested course.
Raises:
- CourseOverview.DoesNotExist if the course specified by course_id
was not found.
- IOError if some other error occurs while trying to load the
course from the module store.
"""
store = modulestore()
with store.bulk_operations(course_id):
course = store.get_course(course_id)
if isinstance(course, CourseDescriptor):
course_overview = cls._create_or_update(course)
try:
with transaction.atomic():
course_overview.save()
# Remove and recreate all the course tabs
CourseOverviewTab.objects.filter(course_overview=course_overview).delete()
CourseOverviewTab.objects.bulk_create([
CourseOverviewTab(tab_id=tab.tab_id, course_overview=course_overview)
for tab in course.tabs
])
# Remove and recreate course images
CourseOverviewImageSet.objects.filter(course_overview=course_overview).delete()
CourseOverviewImageSet.create(course_overview, course)
except IntegrityError:
# There is a rare race condition that will occur if
# CourseOverview.get_from_id is called while a
# another identical overview is already in the process
# of being created.
# One of the overviews will be saved normally, while the
# other one will cause an IntegrityError because it tries
# to save a duplicate.
# (see: https://openedx.atlassian.net/browse/TNL-2854).
pass
except Exception:
log.exception(
"CourseOverview for course %s failed!",
course_id,
)
raise
return course_overview
elif course is not None:
raise IOError(
"Error while loading course {} from the module store: {}",
unicode(course_id),
course.error_msg if isinstance(course, ErrorDescriptor) else unicode(course)
)
else:
raise cls.DoesNotExist()
@classmethod
def get_from_id(cls, course_id):
"""
Load a CourseOverview object for a given course ID.
First, we try to load the CourseOverview from the database. If it
doesn't exist, we load the entire course from the modulestore, create a
CourseOverview object from it, and then cache it in the database for
future use.
Arguments:
course_id (CourseKey): the ID of the course overview to be loaded.
Returns:
CourseOverview: overview of the requested course.
Raises:
- CourseOverview.DoesNotExist if the course specified by course_id
was not found.
- IOError if some other error occurs while trying to load the
course from the module store.
"""
try:
course_overview = cls.objects.select_related('image_set').get(id=course_id)
if course_overview.version < cls.VERSION:
# Throw away old versions of CourseOverview, as they might contain stale data.
course_overview.delete()
course_overview = None
except cls.DoesNotExist:
course_overview = None
# Regenerate the thumbnail images if they're missing (either because
# they were never generated, or because they were flushed out after
# a change to CourseOverviewImageConfig.
if course_overview and not hasattr(course_overview, 'image_set'):
CourseOverviewImageSet.create(course_overview)
return course_overview or cls.load_from_module_store(course_id)
@classmethod
def get_from_ids_if_exists(cls, course_ids):
"""
Return a dict mapping course_ids to CourseOverviews, if they exist.
This method will *not* generate new CourseOverviews or delete outdated
ones. It exists only as a small optimization used when CourseOverviews
are known to exist, for common situations like the student dashboard.
Callers should assume that this list is incomplete and fall back to
get_from_id if they need to guarantee CourseOverview generation.
"""
return {
overview.id: overview
for overview
in cls.objects.select_related('image_set').filter(
id__in=course_ids,
version__gte=cls.VERSION
)
}
@classmethod
def get_from_id_if_exists(cls, course_id):
"""
Return a CourseOverview for the provided course_id if it exists.
Returns None if no CourseOverview exists with the provided course_id
This method will *not* generate new CourseOverviews or delete outdated
ones. It exists only as a small optimization used when CourseOverviews
are known to exist, for common situations like the student dashboard.
Callers should assume that this list is incomplete and fall back to
get_from_id if they need to guarantee CourseOverview generation.
"""
try:
course_overview = cls.objects.select_related('image_set').get(
id=course_id,
version__gte=cls.VERSION
)
except cls.DoesNotExist:
course_overview = None
return course_overview
def clean_id(self, padding_char='='):
"""
Returns a unique deterministic base32-encoded ID for the course.
Arguments:
padding_char (str): Character used for padding at end of base-32
-encoded string, defaulting to '='
"""
return course_metadata_utils.clean_course_key(self.location.course_key, padding_char)
@property
def location(self):
"""
Returns the UsageKey of this course.
UsageKeyField has a strange behavior where it fails to parse the "run"
of a course out of the serialized form of a Mongo Draft UsageKey. This
method is a wrapper around _location attribute that fixes the problem
by calling map_into_course, which restores the run attribute.
"""
if self._location.run is None:
self._location = self._location.map_into_course(self.id)
return self._location
@property
def number(self):
"""
Returns this course's number.
This is a "number" in the sense of the "course numbers" that you see at
lots of universities. For example, given a course
"Intro to Computer Science" with the course key "edX/CS-101/2014", the
course number would be "CS-101"
"""
return course_metadata_utils.number_for_course_location(self.location)
@property
def url_name(self):
"""
Returns this course's URL name.
"""
return block_metadata_utils.url_name_for_block(self)
@property
def display_name_with_default(self):
"""
Return reasonable display name for the course.
"""
return block_metadata_utils.display_name_with_default(self)
@property
def display_name_with_default_escaped(self):
"""
DEPRECATED: use display_name_with_default
Return html escaped reasonable display name for the course.
Note: This newly introduced method should not be used. It was only
introduced to enable a quick search/replace and the ability to slowly
migrate and test switching to display_name_with_default, which is no
longer escaped.
"""
return block_metadata_utils.display_name_with_default_escaped(self)
@property
def dashboard_start_display(self):
"""
Return start date to diplay on learner's dashboard, preferably `Course Advertised Start`
"""
return self.advertised_start or self.start
def has_started(self):
"""
Returns whether the the course has started.
"""
return course_metadata_utils.has_course_started(self.start)
def has_ended(self):
"""
Returns whether the course has ended.
"""
return course_metadata_utils.has_course_ended(self.end)
def has_marketing_url(self):
"""
Returns whether the course has marketing url.
"""
return settings.FEATURES.get('ENABLE_MKTG_SITE') and bool(self.marketing_url)
def has_social_sharing_url(self):
"""
Returns whether the course has social sharing url.
"""
is_social_sharing_enabled = getattr(settings, 'SOCIAL_SHARING_SETTINGS', {}).get('CUSTOM_COURSE_URLS')
return is_social_sharing_enabled and bool(self.social_sharing_url)
def starts_within(self, days):
"""
Returns True if the course starts with-in given number of days otherwise returns False.
"""
return course_metadata_utils.course_starts_within(self.start, days)
@property
def start_date_is_still_default(self):
"""
Checks if the start date set for the course is still default, i.e.
.start has not been modified, and .advertised_start has not been set.
"""
return course_metadata_utils.course_start_date_is_default(
self.start,
self.advertised_start,
)
@property
def sorting_score(self):
"""
Returns a tuple that can be used to sort the courses according
the how "new" they are. The "newness" score is computed using a
heuristic that takes into account the announcement and
(advertised) start dates of the course if available.
The lower the number the "newer" the course.
"""
return course_metadata_utils.sorting_score(self.start, self.advertised_start, self.announcement)
@property
def start_type(self):
"""
Returns the type of the course's 'start' field.
"""
if self.advertised_start:
return u'string'
elif self.start != DEFAULT_START_DATE:
return u'timestamp'
else:
return u'empty'
@property
def start_display(self):
"""
Returns the display value for the course's start date.
"""
if self.advertised_start:
return self.advertised_start
elif self.start != DEFAULT_START_DATE:
return defaultfilters.date(self.start, "DATE_FORMAT")
else:
return None
def may_certify(self):
"""
Returns whether it is acceptable to show the student a certificate
download link.
"""
return course_metadata_utils.may_certify_for_course(
self.certificates_display_behavior,
self.certificates_show_before_end,
self.has_ended(),
self.certificate_available_date,
self.self_paced
)
@property
def pre_requisite_courses(self):
"""
Returns a list of ID strings for this course's prerequisite courses.
"""
return json.loads(self._pre_requisite_courses_json)
@pre_requisite_courses.setter
def pre_requisite_courses(self, value):
"""
Django requires there be a setter for this, but it is not
necessary for the way we currently use it. Due to the way
CourseOverviews are constructed raising errors here will
cause a lot of issues. These should not be mutable after
construction, so for now we just eat this.
"""
pass
@classmethod
def update_select_courses(cls, course_keys, force_update=False):
"""
A side-effecting method that updates CourseOverview objects for
the given course_keys.
Arguments:
course_keys (list[CourseKey]): Identifies for which courses to
return CourseOverview objects.
force_update (boolean): Optional parameter that indicates
whether the requested CourseOverview objects should be
forcefully updated (i.e., re-synched with the modulestore).
"""
log.info('Generating course overview for %d courses.', len(course_keys))
log.debug('Generating course overview(s) for the following courses: %s', course_keys)
action = CourseOverview.load_from_module_store if force_update else CourseOverview.get_from_id
for course_key in course_keys:
try:
action(course_key)
except Exception as ex: # pylint: disable=broad-except
log.exception(
'An error occurred while generating course overview for %s: %s',
unicode(course_key),
text_type(ex),
)
log.info('Finished generating course overviews.')
@classmethod
def get_all_courses(cls, orgs=None, filter_=None):
"""
Returns all CourseOverview objects in the database.
Arguments:
orgs (list[string]): Optional parameter that allows case-insensitive
filtering by organization.
filter_ (dict): Optional parameter that allows custom filtering.
"""
# Note: If a newly created course is not returned in this QueryList,
# make sure the "publish" signal was emitted when the course was
# created. For tests using CourseFactory, use emit_signals=True.
course_overviews = CourseOverview.objects.all()
if orgs:
# In rare cases, courses belonging to the same org may be accidentally assigned
# an org code with a different casing (e.g., Harvardx as opposed to HarvardX).
# Case-insensitive matching allows us to deal with this kind of dirty data.
course_overviews = course_overviews.filter(org__iregex=r'(' + '|'.join(orgs) + ')')
if filter_:
course_overviews = course_overviews.filter(**filter_)
return course_overviews
@classmethod
def get_all_course_keys(cls):
"""
Returns all course keys from course overviews.
"""
return CourseOverview.objects.values_list('id', flat=True)
def is_discussion_tab_enabled(self):
"""
Returns True if course has discussion tab and is enabled
"""
tabs = self.tabs.all()
# creates circular import; hence explicitly referenced is_discussion_enabled
for tab in tabs:
if tab.tab_id == "discussion" and django_comment_client.utils.is_discussion_enabled(self.id):
return True
return False
@property
def image_urls(self):
"""
Return a dict with all known URLs for this course image.
Current resolutions are:
raw = original upload from the user
small = thumbnail with dimensions CourseOverviewImageConfig.current().small
large = thumbnail with dimensions CourseOverviewImageConfig.current().large
If no thumbnails exist, the raw (originally uploaded) image will be
returned for all resolutions.
"""
# This is either the raw image that the course team uploaded, or the
# settings.DEFAULT_COURSE_ABOUT_IMAGE_URL if they didn't specify one.
raw_image_url = self.course_image_url
# Default all sizes to return the raw image if there is no
# CourseOverviewImageSet associated with this CourseOverview. This can
# happen because we're disabled via CourseOverviewImageConfig.
urls = {
'raw': raw_image_url,
'small': raw_image_url,
'large': raw_image_url,
}
# If we do have a CourseOverviewImageSet, we still default to the raw
# images if our thumbnails are blank (might indicate that there was a
# processing error of some sort while trying to generate thumbnails).
if hasattr(self, 'image_set') and CourseOverviewImageConfig.current().enabled:
urls['small'] = self.image_set.small_url or raw_image_url
urls['large'] = self.image_set.large_url or raw_image_url
return self.apply_cdn_to_urls(urls)
@property
def pacing(self):
""" Returns the pacing for the course.
Potential values:
self: Self-paced courses
instructor: Instructor-led courses
"""
return 'self' if self.self_paced else 'instructor'
@property
def closest_released_language(self):
"""
Returns the language code that most closely matches this course' language and is fully
supported by the LMS, or None if there are no fully supported languages that
match the target.
"""
return get_closest_released_language(self.language) if self.language else None
def apply_cdn_to_urls(self, image_urls):
"""
Given a dict of resolutions -> urls, return a copy with CDN applied.
If CDN does not exist or is disabled, just returns the original. The
URLs that we store in CourseOverviewImageSet are all already top level
paths, so we don't need to go through the /static remapping magic that
happens with other course assets. We just need to add the CDN server if
appropriate.
"""
cdn_config = AssetBaseUrlConfig.current()
if not cdn_config.enabled:
return image_urls
base_url = cdn_config.base_url
return {
resolution: self._apply_cdn_to_url(url, base_url)
for resolution, url in image_urls.items()
}
def _apply_cdn_to_url(self, url, base_url):
"""
Applies a new CDN/base URL to the given URL.
If a URL is absolute, we skip switching the host since it could
be a hostname that isn't behind our CDN, and we could unintentionally
break the URL overall.
"""
# The URL can't be empty.
if not url:
return url
_, netloc, path, params, query, fragment = urlparse(url)
# If this is an absolute URL, just return it as is. It could be a domain
# that isn't ours, and thus CDNing it would actually break it.
if netloc:
return url
return urlunparse((None, base_url, path, params, query, fragment))
def __unicode__(self):
"""Represent ourselves with the course key."""
return unicode(self.id)
def test_FloatField(self):
lazy_func = lazy(lambda: 1.2, float)
self.assertIsInstance(
FloatField().get_prep_value(lazy_func()),
float)
class AgencyViewSet(viewsets.ModelViewSet):
"""API views for Agency"""
# pylint: disable=too-many-public-methods
queryset = (Agency.objects.order_by("id").select_related(
"jurisdiction", "parent", "appeal_agency").prefetch_related(
Prefetch(
"emails",
queryset=EmailAddress.objects.filter(
status="good",
agencyemail__request_type="primary",
agencyemail__email_type="to",
),
to_attr="primary_emails",
),
Prefetch(
"phones",
queryset=PhoneNumber.objects.filter(
type="fax",
status="good",
agencyphone__request_type="primary"),
to_attr="primary_faxes",
),
Prefetch(
"addresses",
queryset=Address.objects.filter(
agencyaddress__request_type="primary"),
to_attr="primary_addresses",
),
"types",
).annotate(
average_response_time_=Coalesce(
ExtractDay(
Avg(
F("foiarequest__datetime_done") -
F("foiarequest__composer__datetime_submitted"))),
Value(0),
),
fee_rate_=Coalesce(
100 * CountWhen(foiarequest__price__gt=0,
output_field=FloatField()) /
NullIf(
Count("foiarequest"), Value(0), output_field=FloatField()),
Value(0),
),
success_rate_=Coalesce(
100 * CountWhen(
foiarequest__status__in=["done", "partial"],
output_field=FloatField(),
) / NullIf(
Count("foiarequest"), Value(0), output_field=FloatField()),
Value(0),
),
number_requests=Count("foiarequest"),
number_requests_completed=CountWhen(foiarequest__status="done"),
number_requests_rejected=CountWhen(foiarequest__status="rejected"),
number_requests_no_docs=CountWhen(foiarequest__status="no_docs"),
number_requests_ack=CountWhen(foiarequest__status="ack"),
number_requests_resp=CountWhen(foiarequest__status="processed"),
number_requests_fix=CountWhen(foiarequest__status="fix"),
number_requests_appeal=CountWhen(foiarequest__status="appealing"),
number_requests_pay=CountWhen(foiarequest__status="payment"),
number_requests_partial=CountWhen(foiarequest__status="partial"),
number_requests_lawsuit=CountWhen(foiarequest__status="lawsuit"),
number_requests_withdrawn=CountWhen(
foiarequest__status="abandoned"),
))
serializer_class = AgencySerializer
# don't allow ordering by computed fields
ordering_fields = [
f for f in AgencySerializer.Meta.fields
if f not in ("absolute_url", "average_response_time", "fee_rate",
"success_rate") and not f.startswith(("has_", "number_"))
]
def get_queryset(self):
"""Filter out non-approved agencies for non-staff"""
if self.request.user.is_staff:
return self.queryset
else:
return self.queryset.filter(status="approved")
class Filter(django_filters.FilterSet):
"""API Filter for Agencies"""
jurisdiction = django_filters.NumberFilter(
field_name="jurisdiction__id")
types = django_filters.CharFilter(field_name="types__name",
lookup_expr="iexact")
class Meta:
model = Agency
fields = ("name", "status", "jurisdiction", "types",
"requires_proxy")
filterset_class = Filter
def test_FloatField(self):
self.assertIsInstance(FloatField().get_prep_value(1.2), float)
class Length3D(Func):
"""
ST_3DLENGTH postgis function
"""
function = 'ST_3DLENGTH'
output_field = FloatField()
def _computed_aggregate_field(self):
return FloatField()
def __init__(self, expression, **extra):
output_field = extra.pop('output_field', FloatField())
super(Avg, self).__init__(expression,
output_field=output_field,
**extra)
def __init__(self, expression, sample=False, **extra):
self.function = 'VAR_SAMP' if sample else 'VAR_POP'
super(Variance, self).__init__(expression,
output_field=FloatField(),
**extra)
def __init__(self, expression, sample=False, **extra):
self.function = 'STDDEV_SAMP' if sample else 'STDDEV_POP'
super(StdDev, self).__init__(expression,
output_field=FloatField(),
**extra)
