def percentage_histogram(self):
cnt = MatchPercentHistogram.objects.count()
if cnt == 0:
pct_histo = MatchPercentHistogram.objects.create()
elif cnt > 1:
raise CommandError(u"Too many MatchPercentHistogram objects found in the database.")
else:
pct_histo = MatchPercentHistogram.objects.all()[0]
# E.g. [ {'cnt' 743277, 'pct': Decimal('0')}, ... ]
aggs = list(Match.objects
.filter(percent_churned__lt=15)
.extra(select={'pct': 'round(percent_churned)'})
.values('pct')
.order_by('pct')
.annotate(cnt=Count('pk')))
aggs.sort(key=itemgetter('pct'))
pct_histo.title = "Number of Matches by Overlap Percentage"
pct_histo.bin_axis_label = "Overlap Percentage"
pct_histo.mass_axis_label = "Number of Matches"
pct_histo.save()
def extract_pair(grp):
pct = grp['pct']
try:
pct = u"{}%".format(grp['pct'])
except (ValueError, TypeError):
pass
return (pct, grp['cnt'])
pct_histo.from_sequence((extract_pair(grp)
for grp in aggs),
bincmp=lambda a, b: cmp(natsort_key(a), natsort_key(b)))
def percentage_histogram(self):
cnt = MatchPercentHistogram.objects.count()
if cnt == 0:
pct_histo = MatchPercentHistogram.objects.create()
elif cnt > 1:
raise CommandError(
u"Too many MatchPercentHistogram objects found in the database."
)
else:
pct_histo = MatchPercentHistogram.objects.all()[0]
# E.g. [ {'cnt' 743277, 'pct': Decimal('0')}, ... ]
aggs = list(
Match.objects.filter(percent_churned__lt=15).extra(
select={
'pct': 'round(percent_churned)'
}).values('pct').order_by('pct').annotate(cnt=Count('pk')))
aggs.sort(key=itemgetter('pct'))
pct_histo.title = "Number of Matches by Overlap Percentage"
pct_histo.bin_axis_label = "Overlap Percentage"
pct_histo.mass_axis_label = "Number of Matches"
pct_histo.save()
def extract_pair(grp):
pct = grp['pct']
try:
pct = u"{}%".format(grp['pct'])
except (ValueError, TypeError):
pass
return (pct, grp['cnt'])
pct_histo.from_sequence(
(extract_pair(grp) for grp in aggs),
bincmp=lambda a, b: cmp(natsort_key(a), natsort_key(b)))
def test_natsort_key_public():
# Identical to _natsort_key
# But it raises a deprecation warning
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert natsort_key('a-5.034e2') == _natsort_key('a-5.034e2', key=None, alg=ns.F)
assert len(w) == 1
assert "natsort_key is deprecated as of 3.4.0, please use natsort_keygen" in str(w[-1].message)
assert natsort_key('a-5.034e2', number_type=float, signed=False, exp=False) == _natsort_key('a-5.034e2', key=None, alg=ns.F | ns.U | ns.N)
assert natsort_key('a-5.034e2', alg=ns.F | ns.U | ns.N) == _natsort_key('a-5.034e2', key=None, alg=ns.F | ns.U | ns.N)
# It is called for each element in a list when sorting
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
a = ['a2', 'a5', 'a9', 'a1', 'a4', 'a10', 'a6']
a.sort(key=natsort_key)
assert len(w) == 7
def test_natsort_key_public_raises_DeprecationWarning_when_called():
# But it raises a deprecation warning
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert natsort_key('a-5.034e2') == ('a-', 5, '.', 34, 'e', 2)
assert len(w) == 1
assert "natsort_key is deprecated as of 3.4.0, please use natsort_keygen" in str(w[-1].message)
# It is called for each element in a list when sorting
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
a = ['a2', 'a5', 'a9', 'a1', 'a4', 'a10', 'a6']
a.sort(key=natsort_key)
assert len(w) == 7
def test_natsort_key_public_raises_DeprecationWarning_when_called():
# But it raises a deprecation warning
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert natsort_key("a-5.034e2") == ("a-", 5, ".", 34, "e", 2)
assert len(w) == 1
assert "natsort_key is deprecated as of 3.4.0, please use natsort_keygen" in str(w[-1].message)
# It is called for each element in a list when sorting
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
a = ["a2", "a5", "a9", "a1", "a4", "a10", "a6"]
a.sort(key=natsort_key)
assert len(w) == 7
def test_natsort_key_public_raises_DeprecationWarning_when_called():
# Identical to _natsort_key
# But it raises a deprecation warning
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert natsort_key('a-5.034e2') == _natsort_key('a-5.034e2', key=None, alg=ns.I)
assert len(w) == 1
assert "natsort_key is deprecated as of 3.4.0, please use natsort_keygen" in str(w[-1].message)
# It is called for each element in a list when sorting
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
a = ['a2', 'a5', 'a9', 'a1', 'a4', 'a10', 'a6']
a.sort(key=natsort_key)
assert len(w) == 7
def __init__(self, itemtype, path):
self.itemtype = itemtype
self.path = path
self._path_list = None
self._alphanum = ns.natsort_key(os.path.basename(path), alg=ns.I | ns.LA | ns.IC) if path else ''
self._mimetype = None
try:
self._lstat = os.lstat(self.path)
if stat.S_ISLNK(self._lstat.st_mode):
self._stat = os.stat(self.path)
else:
self._stat = self._lstat
except FileNotFoundError:
self._stat = None
def subject(school, term, subject):
app.mongo.db.schools.find_one_or_404({'fragment': school}, {'_id': True})
c = mongo.SchoolCollections(school)
term = c.term.find_one_or_404(
{
'fragment': term,
'subjects.fragment': subject
}, {
'fragment': True,
'name': True,
'start': True,
'end': True,
'subjects.$': True,
'_id': False,
})
out = term['subjects'][0]
del term['subjects']
out['term'] = term
courses = {segment['id']: [] for segment in out['segments']}
courses[None] = []
for course in c.course.find({'subjects.id': out['id']},
util.project_course.mongo()):
for subject in course['subjects']:
if subject['id'] != out['id']:
continue
segment = subject.get('segment', None)
courses[segment].append(course)
del course['subjects']
for segment in courses.itervalues():
segment.sort(key=lambda (course): natsort.natsort_key(
course['code'], number_type=None))
segments = []
if courses[None]:
segments.append({'courses': courses[None]})
for segment in out['segments']:
if courses[segment['id']]:
segments.append({
'name': segment['name'],
'courses': courses[segment['id']]
})
out['segments'] = segments
return json.jsonify(out)
def sort_and_print_paths(dirs, filterdata, exclude, reverse, number_type):
"""Sort the paths by directoy then by file within that directory.
Print off the results.
"""
number_type = {'digit': None, 'int': int, 'float': float}[number_type]
for dir in natsorted(dirs.keys(), number_type=number_type):
dirs[dir].sort(key=lambda x: natsort_key(x, number_type=number_type))
if reverse:
dirs[dir] = reversed(dirs[dir])
for file in dirs[dir]:
if filterdata is not None:
# Find all the numbers in the filename.
nums = filterdata[2].findall(file)
# If any numbers are between the range, print.
# Otherwise, move to next file.
for num in nums:
if filterdata[0] <= float(num) <= filterdata[1]: break
else:
continue
if exclude and exclude in file: continue
print(os.path.join(dir, file))
def instructor(school, instructor):
app.mongo.db.schools.find_one_or_404({'fragment': school}, {'_id': True})
c = mongo.SchoolCollections(school)
get_term = lambda course: course['term']
instructor = c.instructor.find_one_or_404({'fragment': instructor},
util.project_instructor.mongo())
instructor['name'] = util.name(instructor)
courses = c.course.find({'sections.instructors': instructor['id']},
{'_id': False})
terms = []
courses = sorted(courses, key=get_term)
for term_id, course_iter in itertools.groupby(courses, get_term):
term = c.term.find_one({'id': term_id}, util.project_term.mongo())
assert term
term['course_sections'] = []
term['independent_studies'] = []
for course in course_iter:
for section in course['sections']:
if instructor['id'] not in section['instructors']:
continue
section = util.formatCourseSection(c,
term,
course,
section,
instructors=False)
if course['independent_study']:
term['independent_studies'].append(section)
else:
term['course_sections'].append(section)
for to_sort in (term['course_sections'], term['independent_studies']):
to_sort.sort(key=lambda (section): natsort.natsort_key(
section['course_code'], number_type=None))
terms.append(term)
terms.sort(key=operator.itemgetter('start', 'end'), reverse=True)
out = instructor
out['terms'] = terms
return json.jsonify(out)
def get_releases(package_str, pre=False, key=None, include_hidden=False,
server_url=DEFAULT_SERVER_URL, hidden_url=None):
'''
Query Python Package Index for list of available release for specified
package.
Parameters
----------
package_str : str
Name of package hosted on Python package index. Version constraints are
also supported (e.g., ``"foo", "foo==1.0", "foo>=1.0"``, etc.). See
`version specifiers`_ reference for more details.
pre : bool, optional
Include pre-release packages.
key : function, optional
Key function to sort ``(package_name, release_info)`` items by.
include_hidden : bool, optional
Include "hidden" packages.
server_url : str, optional
URL to JSON API (default=``'https://pypi.python.org/pypi/{}/json'``).
hidden_url : str, optional
URL to XMLRPC API (default=``'https://pypi.python.org/pypi/'`` for PyPI
server URL).
Returns
-------
(string, collections.OrderedDict)
Package name and package release information, indexed by package
version string and ordered by upload time (i.e., most recent release is
last).
.. _version specifiers:
https://www.python.org/dev/peps/pep-0440/#version-specifiers
'''
if all([not include_hidden, hidden_url is None, server_url ==
DEFAULT_SERVER_URL]):
hidden_url = DEFAULT_HIDDEN_URL
if hidden_url is None:
include_hidden = True
match = CRE_PACKAGE.match(package_str)
if not match:
raise ValueError('Invalid package descriptor. Must be like "foo", '
'"foo==1.0", "foo>=1.0", etc.')
package_request = match.groupdict()
response = requests.get(server_url.format(package_request['name']))
package_data = json.loads(response.text)
if not include_hidden:
client = xmlrpclib.ServerProxy(hidden_url)
public_releases = set(client.package_releases(package_request['name']))
if key is None:
key = lambda (k, v): pkg_resources.parse_version(k)
all_releases = OrderedDict(sorted([(k, v[0]) for k, v in
package_data['releases'].iteritems()
if v and (include_hidden or k in
public_releases)], key=key))
if not all_releases:
raise KeyError('No releases found for package: {}'
.format(package_request['name']))
package_request = match.groupdict()
if package_request['version_specifiers']:
comparators = [m.groupdict() for m in CRE_VERSION_SPECIFIERS
.finditer(package_request['version_specifiers'])]
else:
comparators = []
filter_ = lambda v: (True if not comparators
else all(eval('%r %s %r' %
(natsort.natsort_key(v),
c['comparator'],
natsort.natsort_key(c['version'])))
for c in comparators))
# Define regex to check for pre-release.
cre_pre = re.compile(r'\.dev|\.pre')
releases = OrderedDict([(k, v) for k, v in all_releases.iteritems()
if filter_(k) and (pre or not cre_pre.search(k))])
if not releases:
raise KeyError('None of the following releases match the specifiers '
'"{}": {}'.format(package_request['version_specifiers'],
', '.join(all_releases.keys())))
return package_request['name'], releases
def test_natsort_key_public():
assert natsort_key("a-5.034e2") == ("a-", 5, ".", 34, "e", 2)
def test_natsort_key():
a = ['num3', 'num5', 'num2']
a.sort(key=natsort_key)
assert a == ['num2', 'num3', 'num5']
# The below illustrates how the key works, and how the different options affect sorting.
assert natsort_key('a-5.034e1') == ('a', -50.34)
assert natsort_key('a-5.034e1', number_type=float, signed=True,
exp=True) == ('a', -50.34)
assert natsort_key('a-5.034e1', number_type=float, signed=True,
exp=False) == ('a', -5.034, 'e', 1.0)
assert natsort_key('a-5.034e1', number_type=float, signed=False,
exp=True) == ('a-', 50.34)
assert natsort_key('a-5.034e1', number_type=float, signed=False,
exp=False) == ('a-', 5.034, 'e', 1.0)
assert natsort_key('a-5.034e1',
number_type=int) == ('a', -5, '.', 34, 'e', 1)
assert natsort_key('a-5.034e1', number_type=int,
signed=False) == ('a-', 5, '.', 34, 'e', 1)
assert natsort_key('a-5.034e1',
number_type=None) == natsort_key('a-5.034e1',
number_type=int,
signed=False)
# Iterables are parsed recursively so you can sort lists of lists.
assert natsort_key(('a1', 'a10')) == (('a', 1.0), ('a', 10.0))
# Strings that lead with a number get an empty string at the front of the tuple.
# This is designed to get around the "unorderable types" issue.
assert natsort_key(('15a', '6')) == (('', 15.0, 'a'), ('', 6.0))
assert natsort_key(10) == ('', 10)
# Turn on py3_safe to put a '' between adjacent numbers
assert natsort_key('43h7+3',
py3_safe=True) == ('', 43.0, 'h', 7.0, '', 3.0)
# Invalid arguments give the correct response
with raises(ValueError) as err:
natsort_key('a', number_type='float')
assert str(
err.value) == "natsort_key: 'number_type' parameter 'float' invalid"
with raises(ValueError) as err:
natsort_key('a', signed='True')
assert str(err.value) == "natsort_key: 'signed' parameter 'True' invalid"
with raises(ValueError) as err:
natsort_key('a', exp='False')
assert str(err.value) == "natsort_key: 'exp' parameter 'False' invalid"