def stats(request, service_id):
"""Create up some stats."""
service = get_object_or_404(UserService, pk=service_id)
if check_is_service_id(service, MODULE_NAME):
shares = get_items(request.user, date.today() - timedelta(days=7), service)
sources = {}
# count source websites
for share in shares:
if sources.has_key(share.source):
sources[share.source] = sources[share.source] + 1
else:
sources[share.source] = 1
sources = SortedDict(sorted(sources.items(),
reverse=True, key=lambda x: x[1]))
sources_reversed = SortedDict(sorted(sources.items(),
reverse=False, key=lambda x: x[1]))
return render(
request,
{'shares' : shares,
'sources' : sources,
'sources_reversed' : sources_reversed},
service.template_name + '/stats.html'
)
else:
return redirect('/%s' %(request.user.username))
def _html_output(self, form_as, normal_row, help_text_html, sections_re, row_re):
formsets = SortedDict()
for bf in self.formsets:
if bf.label:
label = conditional_escape(force_unicode(bf.label))
# Only add the suffix if the label does not end in
# punctuation.
if self.form.label_suffix:
if label[-1] not in ":?.!":
label += self.form.label_suffix
label = label or ""
else:
label = ""
if bf.field.help_text:
help_text = help_text_html % force_unicode(bf.field.help_text)
else:
help_text = u""
formsets[bf.name] = {"label": force_unicode(label), "field": unicode(bf), "help_text": help_text}
try:
output = []
data = form_as()
section_search = sections_re.search(data)
if formsets:
hidden = u"".join(hidden_re.findall(data))
last_formset_name, last_formset = formsets.items()[-1]
last_formset["field"] = last_formset["field"] + hidden
formsets[last_formset_name] = normal_row % last_formset
for name, formset in formsets.items()[:-1]:
formsets[name] = normal_row % formset
if not section_search:
output.append(data)
else:
section_groups = section_search.groups()
for row, head, item in row_re.findall(section_groups[1]):
head_search = label_re.search(head)
if head_search:
id = head_search.groups()[1]
if formsets.has_key(id):
row = formsets[id]
del formsets[id]
output.append(row)
for name, row in formsets.items():
if name in self.form.fields.keyOrder:
output.append(row)
return mark_safe(u"\n".join(output))
except Exception, e:
import traceback
return traceback.format_exc()
def date_tree():
""" returns a dict in the form {2012: [1,2,3,4,5,6], 2011: [10,11,12]} """
today = datetime.date.today()
first_day = date_of_first_entry()
years = range(first_day.year, today.year+1)
date_tree = SortedDict((year, range(1,13)) for year in years)
for year, months in date_tree.items():
if year == first_day.year:
date_tree[year] = months[(first_day.month-1):]
if year == today.year:
# list might be missing some elements because it is also the first year.
months_this_year = range(1, today.month+1)
date_tree[year] = [m for m in date_tree[year] if m in months_this_year]
return date_tree.items()
def get_extra_context(self, *args, **kwargs):
dls = list(Download.permitted.all())
# create dictionary of categories
cat_dict = SortedDict((id, {'parent': parent,
'title': title,
'items': [],
'subcats': [],
'slug': slug,
'child_count': 0})
for (id, parent, title, slug) in
Category.objects.values_list('id', 'parent',
'title', 'slug'))
# add None key for downloads without a category
cat_dict[None] = {
'parent': None,
'items': [],
'child_count': 0,
'subcats': []
}
# add downloads to category item lists
for dl in dls:
cat_dict[dl.primary_category_id]['items'].append(dl)
# nest subcategories
for key, val in cat_dict.items():
if val['parent']:
cat_dict[val['parent']]['subcats'].append(key)
child_count = len(val['items'])
val['child_count'] += child_count
if child_count > 0:
parent_id = val['parent']
while parent_id:
cat_dict[parent_id]['child_count'] += child_count
parent_id = cat_dict[parent_id]['parent']
# remove categories with no items in them
category_list = []
for key, val in cat_dict.items():
subcats = []
for subcat in val['subcats']:
if cat_dict[subcat]['child_count'] > 0:
subcats.append(cat_dict[subcat])
val['subcats'] = subcats
if val['child_count'] > 0 and not val['parent']:
category_list.append(val)
return {'title': _('Downloads'), 'category_list': category_list}
def get_context_data(self, **kwargs):
context_data = super(PageVisit, self).get_context_data(**kwargs)
if not self.client:
return context_data
query = models.RequestEvent.objects.filter(client=self.client, created__range=[self.start_date, self.end_date])
start_page = self.form.cleaned_data.get("start_page")
if start_page:
query = query.filter(path=start_page)
try:
target_page = query.values("path").annotate(visits=Count("pk")).order_by("-visits")[0]
except IndexError:
return context_data
parent_pages = defaultdict(int)
for parent in models.RequestEvent.objects.filter(
created__range=[self.start_date, self.end_date], path=target_page["path"]
):
key = ""
if parent.referrer:
parsed_url = urlparse(parent.referrer)
key = "%s://%s%s" % (parsed_url.scheme, parsed_url.netloc, parsed_url.path)
parent_pages[key] += 1
parent_pages = SortedDict(sorted(parent_pages.items(), key=lambda x: x[1], reverse=True))
child_pages = (
models.RequestEvent.objects.filter(
created__range=[self.start_date, self.end_date], referrer__endswith=target_page["path"]
)
.values("path")
.annotate(visits=Count("pk"))
.order_by("-visits")
)
page_info = {"target_page": target_page, "parent_pages": parent_pages, "child_pages": child_pages}
context_data.update({"page_info": page_info, "start_page": start_page or target_page["path"]})
return context_data
def get_actions(self):
labels, details = self._review_actions()
actions = SortedDict()
if self.review_type != "preliminary":
actions["public"] = {
"method": self.handler.process_public,
"minimal": False,
"label": _lazy("Push to public"),
}
actions["prelim"] = {"method": self.handler.process_preliminary, "label": labels["prelim"], "minimal": False}
actions["reject"] = {"method": self.handler.process_sandbox, "label": _lazy("Reject"), "minimal": False}
actions["info"] = {
"method": self.handler.request_information,
"label": _lazy("Request more information"),
"minimal": True,
}
actions["super"] = {
"method": self.handler.process_super_review,
"label": _lazy("Request super-review"),
"minimal": True,
}
for k, v in actions.items():
v["details"] = details.get(k)
return actions
def __new__(cls, clsname, bases, attrs):
fields_present = set()
fieldsets = SortedDict()
fieldset_help = {}
fieldset_sets = []
if attrs.has_key('fieldsets'):
fieldset_sets.append(attrs['fieldsets'])
fieldset_sets += [clazz.fieldsets for clazz in bases if hasattr(clazz, 'fieldsets')]
for fieldset in fieldset_sets:
for values in fieldset:
if len(values) == 3:
name, help_text, fields = values
fieldset_help[name] = fieldset_help.get(name, help_text)
else:
name, fields = values
fields = [f for f in fields if f not in fields_present]
fields_present.update(fields)
if not fields:
continue
if fieldsets.has_key(name):
fieldsets[name] = fieldsets[name] + fields
else:
fieldsets[name] = fields
if fieldsets.has_key(None):
del fieldsets[None]
attrs['fieldsets'] = fieldsets.items()
attrs['fieldset_help'] = fieldset_help
return super(FieldsetMetaclass, cls).__new__(cls, clsname, bases, attrs)
class BindingDict(object):
"""
This dict-like object is used to store fields on a serializer.
This ensures that whenever fields are added to the serializer we call
`field.bind()` so that the `field_name` and `parent` attributes
can be set correctly.
"""
def __init__(self, serializer):
self.serializer = serializer
self.fields = SortedDict()
def __setitem__(self, key, field):
self.fields[key] = field
field.bind(field_name=key, parent=self.serializer)
def __getitem__(self, key):
return self.fields[key]
def __delitem__(self, key):
del self.fields[key]
def items(self):
return self.fields.items()
def keys(self):
return self.fields.keys()
def values(self):
return self.fields.values()
def get_stats_items(self, since):
"""Stubbed out for now"""
checkins = self.get_items(since)
categories = {}
mayorships = {}
for checkin in checkins:
if hasattr(checkin, 'is_mayor'):
if mayorships.has_key(checkin.title):
mayorships[checkin.title]['count'] += 1
else:
mayorships[checkin.title] = {'venue' : checkin,
'count' : 1
}
if hasattr(checkin, 'categories'):
if categories.has_key(checkin.categories[0]['name']):
categories[checkin.categories[0]['name']]['count'] += 1
else:
if hasattr(checkin, 'icon'):
icon = checkin.icon
else:
icon = checkin.categories[0]['name']
categories[checkin.categories[0]['name']] = {
'icon' : checkin.icon,
'count' : 1
}
categories = SortedDict(sorted(categories.items(), reverse=True, key=lambda x: x[1]))
return checkins, categories, mayorships
def js_options(self):
options = deepcopy(self._meta.options)
columns = self.bound_columns
aoColumnDefs = options.setdefault('aoColumnDefs', [])
colopts = SortedDict()
for index, name in enumerate(columns.keys()):
column = columns[name]
for key, value in column.options.items():
if not (key, str(value)) in colopts.keys():
colopts[(key, str(value))] = {}
colopts[(key, str(value))]['targets'] = []
colopts[(key, str(value))]['targets'] = colopts[(key, str(value))]['targets'] + [index]
colopts[(key, str(value))]['key'] = key
colopts[(key, str(value))]['value'] = value
if column.sort_field != column.display_field and column.sort_field in columns:
key = 'iDataSort'
value = columns.keys().index(column.sort_field)
if not (key, str(value)) in colopts.keys():
colopts[(key, str(value))] = {}
colopts[(key, str(value))]['targets'] = []
colopts[(key, str(value))]['targets'] = colopts[(key, str(value))]['targets'] + [index]
colopts[(key, str(value))]['key'] = key
colopts[(key, str(value))]['value'] = value
for kv, values in colopts.items():
aoColumnDefs.append(dict([(values['key'], values['value']), ('aTargets', values['targets'])]))
options['sAjaxSource'] = reverse_ajax_source(options.get('sAjaxSource'))
return mark_safe(dumpjs(options, indent=4, sort_keys=True))
def export(self):
swanpardict = SortedDict(self.get_fields())
# clean up the common infomation
del swanpardict['ID']
del swanpardict['name']
del swanpardict['user']
del swanpardict['group']
del swanpardict['time created']
del swanpardict['last modified']
# reconstruct COMPUTE line before deleting
swanpardict['compute'] = "nonst %s %s hr %s" % (
swanpardict['time start'], swanpardict['time interval'], swanpardict['time end'])
del swanpardict['time start']
del swanpardict['time end']
del swanpardict['time interval']
if (self.swanconfig):
swanpar_dir = "%s/%s" % (MEDIA_ROOT, self.swanconfig.model_input.input_dir)
swanpar_file = os.path.abspath('%s/%s' % (swanpar_dir, self.name))
log.into(swanpar_file)
if os.path.exists(swanpar_file):
os.remove(swanpar_file)
f = open(swanpar_file, 'w')
for name, value in swanpardict.items():
f.write("%s %s\n" % (name, value))
f.write('stop')
f.close()
else:
log.warn("no need to export without a model input associated with a parameter file")
def get_language_config(content_language=None):
language = get_language()[:2]
if content_language:
content_language = content_language[:2]
else:
content_language = language
config = {}
config['language'] = language
lang_names = SortedDict()
for lang, name in settings.LANGUAGES:
if lang[:2] not in lang_names: lang_names[lang[:2]] = []
lang_names[lang[:2]].append(_(name))
sp_langs = []
for lang, names in lang_names.items():
if lang == content_language:
default = '+'
else:
default = ''
sp_langs.append(u'%s%s=%s' % (default, ' / '.join(names), lang))
config['spellchecker_languages'] = ','.join(sp_langs)
if content_language in settings.LANGUAGES_BIDI:
config['directionality'] = 'rtl'
else:
config['directionality'] = 'ltr'
if tinymce_settings.USE_SPELLCHECKER:
config['spellchecker_rpc_url'] = reverse('tinymce.views.spell_check')
return config
def get_favored_results(self, error=5, threshhold=None): """ Calculates the set of most-favored results based on their weight. Evenly-weighted results will be grouped together and either added or excluded as a group. :param error: An arbitrary number; higher values will cause this method to be more reticent about adding new items to the favored results. :param threshhold: Will be passed directly into :meth:`get_weighted_results` """ if not hasattr(self, '_favored_results'): results = self.get_weighted_results(threshhold) grouped_results = SortedDict() for result in results: grouped_results.setdefault(result.weight, []).append(result) self._favored_results = [] for value, subresults in grouped_results.items(): cost = error * sum([(value - result.weight)**2 for result in self._favored_results]) if value > cost: self._favored_results += subresults else: break if len(self._favored_results) == len(results): self._favored_results = [] return self._favored_results
def SearchCode(self, event): # wxGlade: ReadCodeBrowser.<event_handler>
#get all matching codes
search_results_header = 'All other codes found...'
search_codes_list = list(ReadCode.objects.filter(concept_status='C'
).filter(synonyms__term_30__icontains=self.search_text.GetValue()
).distinct())
code_id_list = [x.code for x in search_codes_list]
final_codes = SortedDict()
for (k,v) in self.preferred_codes:
final_codes[k] = []
final_codes[search_results_header] = []
font = self.search_results.GetFont()
font.SetWeight(wx.FONTWEIGHT_BOLD)
for c in search_codes_list:
#first look in the preferred codes, and add it to that
for (k,v) in self.preferred_codes:
if c.code in v:
final_codes[k].append(c)
#now look to see if it has any parents in the rest of the codes
parents = set(c.get_ancestor_codes().split(','))
parents.discard(c.code)
for parent in parents:
if parent in code_id_list:
#this code has a matching parent - do not include
break
else:
final_codes[search_results_header].append(c)
self.search_results.Clear()
for (k,v) in final_codes.items():
if final_codes[k]:
item = self.search_results.Append(k)
self.search_results.SetItemFont(item,font)
for f in final_codes[k]:
self.search_results.Append(unicode(f),f)
def js_options(self):
options = deepcopy(self._meta.options)
aoColumnDefs = options.setdefault('aoColumnDefs', [])
colopts = SortedDict()
for index, bcol in enumerate(self.bound_columns.values()):
for key, value in bcol.options.items():
if not (key, str(value)) in colopts.keys():
colopts[(key, str(value))] = {}
colopts[(key, str(value))]['targets'] = []
coltargets = colopts[(key, str(value))]['targets'] + [index]
colopts[(key, str(value))]['targets'] = coltargets
colopts[(key, str(value))]['key'] = key
colopts[(key, str(value))]['value'] = value
if bcol.sort_field not in self.bound_columns:
continue
if bcol.sort_field == bcol.display_field:
continue
key = 'iDataSort'
value = self.bound_columns.keys().index(bcol.sort_field)
if not (key, str(value)) in colopts.keys():
colopts[(key, str(value))] = {}
colopts[(key, str(value))]['targets'] = []
coltargets = colopts[(key, str(value))]['targets'] + [index]
colopts[(key, str(value))]['targets'] = coltargets
colopts[(key, str(value))]['key'] = key
colopts[(key, str(value))]['value'] = value
for kv, values in colopts.items():
aoColumnDefs.append(dict([(values['key'], values['value']),
('aTargets', values['targets'])]))
return mark_safe(dumpjs(options, indent=4, sort_keys=True))
def get_actions(self, request, addon):
labels, details = self._review_actions()
actions = SortedDict()
if not addon.admin_review or acl.action_allowed(request, "ReviewerAdminTools", "View"):
if self.review_type != "preliminary":
actions["public"] = {
"method": self.handler.process_public,
"minimal": False,
"label": _lazy("Push to public"),
}
actions["prelim"] = {
"method": self.handler.process_preliminary,
"label": labels["prelim"],
"minimal": False,
}
actions["reject"] = {"method": self.handler.process_sandbox, "label": _lazy("Reject"), "minimal": False}
actions["info"] = {
"method": self.handler.request_information,
"label": _lazy("Request more information"),
"minimal": True,
}
actions["super"] = {
"method": self.handler.process_super_review,
"label": _lazy("Request super-review"),
"minimal": True,
}
actions["comment"] = {"method": self.handler.process_comment, "label": _lazy("Comment"), "minimal": True}
for k, v in actions.items():
v["details"] = details.get(k)
return actions
class ZipCodeFilterSpec(FilterSpec):
def __init__(self, request, params, model, model_admin):
super(ZipCodeFilterSpec, self).__init__(request, params, model, model_admin)
self.links = SortedDict((("Any", "zip_any"), ("90***", "zip_90"), ("75***", "zip_75")))
def consumed_params(self):
return self.links.values()
def choices(self, cl):
selected = [v for v in self.links.values() if self.params.has_key(v)]
for title, key in self.links.items():
yield {
"selected": (self.params.has_key(key) or (key == "zip_any" and selected == [])),
"query_string": cl.get_query_string({key: 1}, selected),
"display": title,
}
def get_query_set(self, cl, qs):
if self.params.has_key("zip_90"):
return qs.filter(locations__zip_code__startswith="90").distinct()
if self.params.has_key("zip_75"):
return qs.filter(locations__zip_code__startswith="75").distinct()
return qs
def title(self):
return u"Zip code"
class CompanyOpenFilterSpec(FilterSpec):
def __init__(self, request, params, model, model_admin):
super(CompanyOpenFilterSpec, self).__init__(request, params, model, model_admin)
self.links = SortedDict((("Any", "open_any"), ("Weekdays", "open_weedays"), ("Weekends", "open_weekends")))
def consumed_params(self):
return self.links.values()
def choices(self, cl):
selected = [v for v in self.links.values() if self.params.has_key(v)]
for title, key in self.links.items():
yield {
"selected": (self.params.has_key(key) or (key == "open_any" and selected == [])),
"query_string": cl.get_query_string({key: 1}, selected),
"display": title,
}
def get_query_set(self, cl, qs):
if self.params.has_key("open_weedays"):
return qs.filter(locations__open_days__contains="12345").distinct()
if self.params.has_key("open_weekends"):
return qs.filter(locations__open_days__contains="67").distinct()
return qs
def title(self):
return u"Locations open?"
def stats(request, service_id):
"""Create up some stats.
"""
service = get_object_or_404(UserService, pk=service_id)
if check_is_service_id(service, PACKAGE_NAME):
shares = service.handler.get_items(date.today() - timedelta(days=7))
sources = {}
# Count source websites
for share in shares:
if sources.has_key(share.source):
sources[share.source] = sources[share.source] + 1
else:
sources[share.source] = 1
sources = SortedDict(
sorted(sources.items(), reverse=True, key=lambda x: x[1])
)
return render(
request,
{
'shares' : shares,
'sources' : sources,
},
'causal/googlereader/stats.html'
)
else:
return redirect('/%s' % (request.user.username,))
def get_language_config(content_language=None):
language = get_language()[:2]
if content_language:
content_language = content_language[:2]
else:
content_language = language
config = {}
config["language"] = language
lang_names = SortedDict()
for lang, name in settings.LANGUAGES:
if lang[:2] not in lang_names:
lang_names[lang[:2]] = []
lang_names[lang[:2]].append(_(name))
sp_langs = []
for lang, names in lang_names.items():
if lang == content_language:
default = "+"
else:
default = ""
sp_langs.append(u"%s%s=%s" % (default, " / ".join(names), lang))
config["spellchecker_languages"] = ",".join(sp_langs)
if content_language in settings.LANGUAGES_BIDI:
config["directionality"] = "rtl"
else:
config["directionality"] = "ltr"
if tinymce.settings.USE_SPELLCHECKER:
config["spellchecker_rpc_url"] = reverse("tinymce.views.spell_check")
return config
def dump_data(request,appname):
app_list = SortedDict()
try:
if request.POST:
for appname in request.POST.getlist('apps'):
app = get_app(appname)
app_list[app] = None
appname = 'choices'
else:
app = get_app(appname)
app_list[app] = None
except ImproperlyConfigured:
if appname == 'all':
for app in get_apps():
app_list[app] = None
if(len(app_list) > 0):
objects = []
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_syncdb(DEFAULT_DB_ALIAS, model):
objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all())
serializers.get_serializer('json')
json = serializers.serialize('json', objects, indent=2,use_natural_keys=True)
response = HttpResponse(json, mimetype='application/json');
response['Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (date.today().__str__(),appname)
return response
return render_to_response('diagnostic/dumpdata.html',context_instance=RequestContext(request))
def history(request):
"""
Tally total expenses and income for each month
"""
history = SortedDict()
entries = Entry.objects.all().order_by('-date')
for e in entries:
# Create dict key
this_month = datetime.date(e.date.year, e.date.month, 1)
if not history.has_key(this_month):
history[this_month] = {'income':0, 'expense':0}
#sum values for month
if e.category.type in ['EXP', 'COGS']:
history[this_month]['expense'] += e.amount
elif e.category.type == 'INC':
history[this_month]['income'] += e.amount
for date, value_dict in history.items():
value_dict['net'] = value_dict['income'] - value_dict['expense']
return simple.direct_to_template(request,
template='beancounter/history.html',
extra_context = { 'history': history })
def get_context_data(self, **kwargs):
context_data = super(Referrer, self).get_context_data(**kwargs)
if not self.client:
return context_data
referrer_query = models.RequestEvent.objects.filter(
client=self.client,
created__range=[self.start_date, self.end_date]
).exclude(referrer=None).exclude(referrer='')
for domain in self.client.domain_set.all():
referrer_query = referrer_query.exclude(
referrer__regex='^https?://.*%s/?.*' % domain.pattern
)
referrer_query = referrer_query.values('referrer').annotate(
referrals=Count('pk')
)
referrer_data = defaultdict(int)
for data in referrer_query:
parse_result = urlparse(data['referrer'])
path = parse_result.path or '/'
referrer_data[
'%s://%s%s' % (
parse_result.scheme, parse_result.netloc, path
)
] += data['referrals']
referrer_data = SortedDict(
sorted(referrer_data.items(), key=lambda x: x[1], reverse=True)
)
context_data['referrer_data'] = referrer_data
return context_data
def get_context_data(self, **kwargs):
context_data = super(ExitPage, self).get_context_data(**kwargs)
if not self.client:
return context_data
exit_page_data = defaultdict(int)
sessions_query = models.RequestEvent.objects.values(
'tracking_key'
).annotate(new_date=Min('created')).filter(
new_date__range=[self.start_date, self.end_date],
client=self.client
)
sessions = list(data['tracking_key'] for data in sessions_query)
for exit_page_row in models.RequestEvent.objects.filter(
tracking_key__in=sessions
).values('path', 'tracking_key').order_by(
'tracking_key', '-created'
):
if exit_page_row['tracking_key'] in sessions:
exit_page_data[exit_page_row['path']] += 1
sessions.pop(sessions.index(exit_page_row['tracking_key']))
exit_page_data = SortedDict(
sorted(exit_page_data.items(), key=lambda x: x[1], reverse=True)
)
context_data.update({
'exit_page_data': exit_page_data
})
return context_data
def build_kml(view, kml_type, id, kml_args_dict):
'''builds a dynamic KML file'''
# n.b.: kml_args_dict containts direct USER input
# This means that we should process input as unsafe in the factory methods...
# ensure kml_args_dict is sorted, so URLS always come out the same
# this help Google Earth's caching
kml_args_dict = SortedDict(sorted(kml_args_dict.items()))
template_context = {
'view': view,
'kml_args': urllib.urlencode(kml_args_dict)
}
if kml_type == 'transect' and id is not None:
transect = makejarkustransect(id)
extra_context = build_transect_context(transect, kml_args_dict)
template_context.update(extra_context)
elif kml_type == 'lod':
lod = makejarkuslod()
extra_context = build_lod_context(lod, kml_args_dict)
template_context.update(extra_context)
elif kml_type == 'style':
extra_context = build_style_context(kml_args_dict)
template_context.update(extra_context)
else:
raise Exception('KML type not supported')
return render_to_kmz("kml/{0}.kml".format(kml_type), template_context)
def get_actions(self):
labels, details = self._review_actions()
actions = SortedDict()
if self.review_type != 'preliminary':
actions['public'] = {'method': self.handler.process_public,
'minimal': False,
'label': _lazy('Push to public')}
actions['prelim'] = {'method': self.handler.process_preliminary,
'label': labels['prelim'],
'minimal': False}
actions['reject'] = {'method': self.handler.process_sandbox,
'label': _lazy('Reject'),
'minimal': False}
actions['info'] = {'method': self.handler.request_information,
'label': _lazy('Request more information'),
'minimal': True}
actions['super'] = {'method': self.handler.process_super_review,
'label': _lazy('Request super-review'),
'minimal': True}
actions['comment'] = {'method': self.handler.process_comment,
'label': _lazy('Comment'),
'minimal': True}
for k, v in actions.items():
v['details'] = details.get(k)
return actions
def make_basic_chart_json(
values,
chart_type='pie',
chart_tip='#val#',
chart_nolabels=True,
chart_make_value=default_make_value,
chart_params=None,
elements_params=None,
):
# Sorts from biggest to smallest
values = values.items()
values.sort(lambda a,b: cmp(b[1], a[1]))
values = SortedDict([(k,v) for k,v in values if v])
data = {"bg_colour": "#ffffff",
"elements": [{
"type": chart_type,
"tip": _(chart_tip),
"no-labels": chart_nolabels,
"colours": [random_color() for i in values.values()],
"values": [chart_make_value(num, i) for num, i in enumerate(values.items())],
"alpha": 0.7,
}],
}
if chart_params:
data.update(chart_params)
if elements_params:
for e in data['elements']:
e.update(elements_params)
return HttpResponse(simplejson.dumps(data, cls=BIEncoder), mimetype="text/javascript")
def get_actions(self, request, addon):
actions = SortedDict()
if request is None:
# If request is not set, it means we are just (ab)using the
# ReviewHelper for its `handler` attribute and we don't care about
# the actions.
return actions
labels, details = self._review_actions()
reviewable_because_complete = addon.status not in (
amo.STATUS_NULL, amo.STATUS_DELETED)
reviewable_because_admin = (
not addon.admin_review or
acl.action_allowed(request, 'ReviewerAdminTools', 'View'))
reviewable_because_submission_time = (
not is_limited_reviewer(request) or
(addon.latest_version is not None and
addon.latest_version.nomination is not None and
(datetime.datetime.now() - addon.latest_version.nomination >=
datetime.timedelta(hours=REVIEW_LIMITED_DELAY_HOURS))))
reviewable_because_pending = addon.latest_version is not None and (
len(addon.latest_version.is_unreviewed) > 0 or
addon.status == amo.STATUS_LITE_AND_NOMINATED)
if (reviewable_because_complete and
reviewable_because_admin and
reviewable_because_submission_time and
reviewable_because_pending):
if self.review_type != 'preliminary':
if addon.is_listed:
label = _lazy('Push to public')
else:
label = _lazy('Grant full review')
actions['public'] = {'method': self.handler.process_public,
'minimal': False,
'label': label}
# An unlisted sideload add-on, which requests a full review, cannot
# be granted a preliminary review.
prelim_allowed = not waffle.flag_is_active(
request, 'no-prelim-review') and addon.is_listed
if prelim_allowed or self.review_type == 'preliminary':
actions['prelim'] = {
'method': self.handler.process_preliminary,
'label': labels['prelim'],
'minimal': False}
actions['reject'] = {'method': self.handler.process_sandbox,
'label': _lazy('Reject'),
'minimal': False}
actions['info'] = {'method': self.handler.request_information,
'label': _lazy('Request more information'),
'minimal': True}
actions['super'] = {'method': self.handler.process_super_review,
'label': _lazy('Request super-review'),
'minimal': True}
actions['comment'] = {'method': self.handler.process_comment,
'label': _lazy('Comment'),
'minimal': True}
for k, v in actions.items():
v['details'] = details.get(k)
return actions
def get_external_url(self):
parts = urlparse.urlparse(self.external_url)
# for avoiding IE errors we have to parse the URL query string to set wmode=transparent
scheme, netloc, path, params, query, fragment = parts[:6]
querydict = SortedDict(parse_qsl(query))
querydict['wmode'] = 'transparent'
query = '&'.join(['%s=%s' % (key, value) for (key, value) in querydict.items()])
return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
def log_header(self, *args, **kwargs):
if not self._log_headers:
return ''
lhd = SortedDict([(k, v) for k, v in self.log_header_dict(*args, **kwargs).items() if v is not None])
if not lhd:
return ''
header = '|'.join('%s:%s' % item for item in lhd.items())
return '{%s} ' % (header.strip())
def get_actions(self):
labels, details = self._review_actions()
actions = SortedDict()
if self.review_type != 'preliminary':
actions['public'] = {
'method': self.handler.process_public,
'minimal': False,
'label': _lazy('Push to public')
}
if not self.addon.is_premium():
actions['prelim'] = {
'method': self.handler.process_preliminary,
'label': labels['prelim'],
'minimal': False
}
actions['reject'] = {
'method': self.handler.process_sandbox,
'label': _lazy('Reject'),
'minimal': False
}
actions['info'] = {
'method': self.handler.request_information,
'label': _lazy('Request more information'),
'minimal': True
}
actions['super'] = {
'method': self.handler.process_super_review,
'label': _lazy('Request super-review'),
'minimal': True
}
actions['comment'] = {
'method': self.handler.process_comment,
'label': _lazy('Comment'),
'minimal': True
}
for k, v in actions.items():
v['details'] = details.get(k)
return actions
def get_fields(self):
"""
Returns the complete set of fields for the object as a dict.
This will be the set of any explicitly declared fields,
plus the set of fields returned by get_default_fields().
"""
ret = SortedDict()
# Get the explicitly declared fields
base_fields = copy.deepcopy(self.base_fields)
for key, field in base_fields.items():
ret[key] = field
# Add in the default fields
default_fields = self.get_default_fields()
for key, val in default_fields.items():
if key not in ret:
ret[key] = val
# If 'fields' is specified, use those fields, in that order.
if self.opts.fields:
assert isinstance(
self.opts.fields,
(list, tuple)), '`fields` must be a list or tuple'
new = SortedDict()
for key in self.opts.fields:
new[key] = ret[key]
ret = new
# Remove anything in 'exclude'
if self.opts.exclude:
assert isinstance(
self.opts.exclude,
(list, tuple)), '`exclude` must be a list or tuple'
for key in self.opts.exclude:
ret.pop(key, None)
for key, field in ret.items():
field.initialize(parent=self, field_name=key)
return ret
def _performance_by_month(user_id, months=12, end_month=None, end_year=None):
monthly_data = SortedDict()
now = datetime.now()
if not end_month:
end_month = now.month
if not end_year:
end_year = now.year
end_time = time.mktime((end_year, end_month + 1, 1, 0, 0, 0, 0, 0, -1))
start_time = time.mktime((end_year, end_month + 1 - months,
1, 0, 0, 0, 0, 0, -1))
sql = (PerformanceGraph.objects
.filter_raw('log_activity.created >=',
date.fromtimestamp(start_time).isoformat())
.filter_raw('log_activity.created <',
date.fromtimestamp(end_time).isoformat())
)
for row in sql.all():
label = row.approval_created.isoformat()[:7]
if not label in monthly_data:
xaxis = row.approval_created.strftime('%b %Y')
monthly_data[label] = dict(teamcount=0, usercount=0,
teamamt=0, label=xaxis)
monthly_data[label]['teamamt'] = monthly_data[label]['teamamt'] + 1
monthly_data_count = monthly_data[label]['teamcount']
monthly_data[label]['teamcount'] = monthly_data_count + row.total
if row.user_id == user_id:
user_count = monthly_data[label]['usercount']
monthly_data[label]['usercount'] = user_count + row.total
# Calculate averages
for i, vals in monthly_data.items():
average = round(vals['teamcount'] / float(vals['teamamt']), 1)
monthly_data[i]['teamavg'] = str(average) # floats aren't valid json
return monthly_data
def profile(request, username, template="accounts/account_profile.html"):
"""
Display a profile.
"""
profile_user = get_object_or_404(User, username=username, is_active=True)
profile_fields = SortedDict()
Profile = get_profile_model()
if Profile is not None:
profile = profile_user.get_profile()
user_fieldname = get_profile_user_fieldname()
exclude = tuple(settings.ACCOUNTS_PROFILE_FORM_EXCLUDE_FIELDS)
for field in Profile._meta.fields:
if field.name not in ("id", user_fieldname) + exclude:
value = getattr(profile, field.name)
profile_fields[field.verbose_name.title()] = value
context = {
"profile_user": profile_user,
"profile_fields": profile_fields.items(),
}
return render(request, template, context)
def profile_fields(user):
"""
Returns profile fields as a dict for the given user. Used in the
profile view template when the ``ACCOUNTS_PROFILE_VIEWS_ENABLED``
setting is set to ``True``, and also in the account approval emails
sent to administrators when the ``ACCOUNTS_APPROVAL_REQUIRED``
setting is set to ``True``.
"""
fields = SortedDict()
try:
profile = get_profile_for_user(user)
user_fieldname = get_profile_user_fieldname()
exclude = tuple(settings.ACCOUNTS_PROFILE_FORM_EXCLUDE_FIELDS)
for field in profile._meta.fields:
if field.name not in ("id", user_fieldname) + exclude:
value = getattr(profile, field.name)
fields[field.verbose_name.title()] = value
except ProfileNotConfigured:
pass
return list(fields.items())
def extra_context(self):
"""
Allows the addition of more context variables as needed.
Must return a dictionary.
"""
documents = SortedDict()
for r in self.results:
if r.document_id in documents:
documents[r.document_id]['pages'].append(r.object)
else:
documents[r.document_id] = {
'id': r.object.document.id,
'document': r.object.document.document,
'pages': [r.object]
}
paginator = Paginator(documents.items(), 5)
try:
page = self.request.GET.get('pag')
docs = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
docs = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999),
# deliver last page of results.
docs = paginator.page(paginator.num_pages)
cp = self.request.GET.copy()
if 'pag' in cp:
cp.pop('pag')
return {
'docs': docs,
'total': len(documents),
'vs_query': self.vs_query,
'refs_fields': None,
'url_query': cp.urlencode,
'tags': json.dumps([tag.name for tag in Tag.objects.all()]),
}
def data_dump(request):
if not request.user.is_authenticated() or not request.user.is_superuser:
response = HttpResponse(json.dumps({"Error": "Not Authorized"}))
response['Content-type'] = 'application/json'
return response
#if wanted all apps, but only want campus
#app_list = SortedDict([(app, None) for app in get_apps()])
app_list = SortedDict([(get_app('campus'), None)])
# Now collate the objects to be serialized.
objects = []
# Needed because sqllite doesn't use
def ordering(self):
if hasattr(self, 'name'):
return str(self.name).lower()
elif hasattr(self, 'id'):
return self.id
else:
return self.pk
for model in sort_dependencies(app_list.items()):
# skip groupedlocation model (not needed since Group uses natural keys)
if model == GroupedLocation:
continue
# - make ordering case insensitive
# - must also make special case for MapObj else the leaf class will be
# serialized, not the actual MapObj itself
if model == MapObj:
objects.extend(sorted(model.objects.mob_filter(), key=ordering))
else:
objects.extend(sorted(model.objects.all(), key=ordering))
try:
data = serializers.serialize('json',
objects,
indent=4,
use_natural_keys=True)
except Exception, e:
data = serializers.serialize('json', "ERORR!")
def get_language_config(content_language=None):
"""Get language config."""
language = get_language()[:2]
if content_language:
content_language = content_language[:2]
else:
content_language = language
config = {}
config['language'] = language
lang_names = SortedDict()
for lang, name in settings.LANGUAGES:
if lang[:2] not in lang_names:
lang_names[lang[:2]] = []
lang_names[lang[:2]].append(_(name))
sp_langs = []
for lang, names in lang_names.items():
if lang == content_language:
default = '+'
else:
default = ''
sp_langs.append('%s%s=%s' % (default, ' / '.join(names), lang))
config['spellchecker_languages'] = ','.join(sp_langs)
if content_language in settings.LANGUAGES_BIDI:
config['directionality'] = 'rtl'
else:
config['directionality'] = 'ltr'
if tinymce_settings.USE_SPELLCHECKER:
config['spellchecker_rpc_url'] = reverse('tinymce.views.spell_check')
return config
def __call__(self, request, tab_class=None):
interface = self.interface_class()
tabs = SortedDict()
for t in interface.tabs:
tab = t()
if tab_class == t and request.method == 'POST':
form = tab.form(request.POST, prefix=tab.form_prefix)
if form.is_valid():
cleaned_data = form.cleaned_data
display_type = cleaned_data.pop('display')
return ReportResponse(tab.report,
request,
display_type,
**cleaned_data)
else:
form = tab.form(prefix=tab.form_prefix)
tabs[tab] = form
ctx = {'tabs': tabs.items(), 'interface': interface, 'name': self.title}
return render(request, 'reporting/interface.html', ctx)
def faceted_filters(self):
url = QueryURLObject(self.url)
filter_mapping = SortedDict(
(filter_['slug'], filter_) for filter_ in self.serialized_filters)
filter_groups = SortedDict()
for slug, facet in self.facet_counts().items():
if not isinstance(facet, dict):
# let's just blankly ignore any non-filter or non-query filters
continue
filter_ = filter_mapping.get(slug, None)
if filter_ is None:
name = slug
group = None
else:
# Let's check if we can get the name from the gettext catalog
name = _(filter_['name'])
group = _(filter_['group']['name'])
f = Filter(url=url,
page=self.current_page,
name=name,
slug=slug,
count=facet.get('count', 0),
active=slug in self.topics,
group=group)
filter_groups.setdefault((group, filter_['group']['order']),
[]).append(f)
# return a sorted list of filters here
grouped_filters = []
for (group, order), filters in filter_groups.items():
sorted_filters = sorted(filters, key=attrgetter('name'))
grouped_filters.append(FilterGroup(group, order, sorted_filters))
return sorted(grouped_filters, key=attrgetter('order'), reverse=True)
def get_actions(self, request, addon):
labels, details = self._review_actions()
actions = SortedDict()
if not addon.admin_review or acl.action_allowed(
request, 'ReviewerAdminTools', 'View'):
if self.review_type != 'preliminary':
if addon.is_listed:
label = _lazy('Push to public')
else:
label = _lazy('Grant full review')
actions['public'] = {'method': self.handler.process_public,
'minimal': False,
'label': label}
# An unlisted sideload add-on, which requests a full review, cannot
# be granted a preliminary review.
if addon.is_listed or self.review_type == 'preliminary':
actions['prelim'] = {
'method': self.handler.process_preliminary,
'label': labels['prelim'],
'minimal': False}
actions['reject'] = {'method': self.handler.process_sandbox,
'label': _lazy('Reject'),
'minimal': False}
actions['info'] = {'method': self.handler.request_information,
'label': _lazy('Request more information'),
'minimal': True}
actions['super'] = {'method': self.handler.process_super_review,
'label': _lazy('Request super-review'),
'minimal': True}
actions['comment'] = {'method': self.handler.process_comment,
'label': _lazy('Comment'),
'minimal': True}
for k, v in actions.items():
v['details'] = details.get(k)
return actions
def dump_data(request, appname):
app_list = SortedDict()
try:
if request.method == 'POST':
for appname in request.POST.getlist('apps'):
app = get_app(appname)
app_list[app] = None
appname = 'choices'
else:
app = get_app(appname)
app_list[app] = None
except ImproperlyConfigured:
if appname == 'all':
for app in get_apps():
app_list[app] = None
if (len(app_list) > 0):
objects = []
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_syncdb(
DEFAULT_DB_ALIAS, model):
objects.extend(
model._default_manager.using(DEFAULT_DB_ALIAS).all())
serializers.get_serializer('json')
json = serializers.serialize('json',
objects,
indent=2,
use_natural_keys=True)
response = HttpResponse(json, mimetype='application/json')
response[
'Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (
date.today().__str__(), appname)
return response
return render_to_response('diagnostic/dumpdata.html',
context_instance=RequestContext(request))
def export(self):
swanpardict = SortedDict(self.get_fields())
# clean up the common infomation
del swanpardict['ID']
del swanpardict['name']
del swanpardict['user']
del swanpardict['group']
del swanpardict['time created']
del swanpardict['last modified']
# reconstruct COMPUTE line before deleting
swanpardict['compute'] = "nonst %s %s hr %s" % (
swanpardict['time start'], swanpardict['time interval'],
swanpardict['time end'])
del swanpardict['time start']
del swanpardict['time end']
del swanpardict['time interval']
if (self.swanconfig):
swanpar_dir = "%s/%s" % (MEDIA_ROOT,
self.swanconfig.model_input.input_dir)
swanpar_file = os.path.abspath('%s/%s' % (swanpar_dir, self.name))
log.into(swanpar_file)
if os.path.exists(swanpar_file):
os.remove(swanpar_file)
f = open(swanpar_file, 'w')
for name, value in swanpardict.items():
f.write("%s %s\n" % (name, value))
f.write('stop')
f.close()
else:
log.warn(
"no need to export without a model input associated with a parameter file"
)
class SMSChoicesNumEnum(ChoicesNumEnum):
def __init__(self, *items):
"""
Receives item with four values key, label, i and send state choices
"""
super(SMSChoicesNumEnum,
self).__init__(*((key, label, i) for key, label, i, _ in items))
self.sender_enum = SortedDict()
for (key, label, i, sender_choices) in items:
for j, choice_label in sender_choices:
self.sender_enum[j] = (choice_label, i)
@property
def sender_choices(self):
return [(val, choice[0]) for val, choice in self.sender_enum.items()]
def get_value_from_sender_value(self, sender_val):
"""
Return value according to sender_val
"""
return self.sender_enum.get(
sender_val)[1] if sender_val in self.sender_enum else self.ERROR
def get_actions(self, request, addon):
actions = SortedDict()
if request is None:
# If request is not set, it means we are just (ab)using the
# ReviewHelper for its `handler` attribute and we don't care about
# the actions.
return actions
labels, details = self._review_actions()
reviewable_because_complete = addon.status not in (amo.STATUS_NULL,
amo.STATUS_DELETED)
reviewable_because_admin = (not addon.admin_review
or acl.action_allowed(
request, 'ReviewerAdminTools', 'View'))
reviewable_because_submission_time = (
not is_limited_reviewer(request)
or (addon.latest_version is not None
and addon.latest_version.nomination is not None and
(datetime.datetime.now() - addon.latest_version.nomination >=
datetime.timedelta(hours=REVIEW_LIMITED_DELAY_HOURS))))
reviewable_because_pending = addon.latest_version is not None and len(
addon.latest_version.is_unreviewed) > 0
if (reviewable_because_complete and reviewable_because_admin
and reviewable_because_submission_time
and reviewable_because_pending):
if self.review_type != 'preliminary':
if addon.is_listed:
label = _lazy('Push to public')
else:
label = _lazy('Grant full review')
actions['public'] = {
'method': self.handler.process_public,
'minimal': False,
'label': label
}
# An unlisted sideload add-on, which requests a full review, cannot
# be granted a preliminary review.
if addon.is_listed or self.review_type == 'preliminary':
actions['prelim'] = {
'method': self.handler.process_preliminary,
'label': labels['prelim'],
'minimal': False
}
actions['reject'] = {
'method': self.handler.process_sandbox,
'label': _lazy('Reject'),
'minimal': False
}
actions['info'] = {
'method': self.handler.request_information,
'label': _lazy('Request more information'),
'minimal': True
}
actions['super'] = {
'method': self.handler.process_super_review,
'label': _lazy('Request super-review'),
'minimal': True
}
actions['comment'] = {
'method': self.handler.process_comment,
'label': _lazy('Comment'),
'minimal': True
}
for k, v in actions.items():
v['details'] = details.get(k)
return actions
class TableTab(Tab):
"""
A :class:`~horizon.tabs.Tab` class which knows how to deal with
:class:`~horizon.tables.DataTable` classes rendered inside of it.
This distinct class is required due to the complexity involved in handling
both dynamic tab loading, dynamic table updating and table actions all
within one view.
.. attribute:: table_classes
An iterable containing the :class:`~horizon.tables.DataTable` classes
which this tab will contain. Equivalent to the
:attr:`~horizon.tables.MultiTableView.table_classes` attribute on
:class:`~horizon.tables.MultiTableView`. For each table class you
need to define a corresponding ``get_{{ table_name }}_data`` method
as with :class:`~horizon.tables.MultiTableView`.
"""
table_classes = None
def __init__(self, tab_group, request):
super(TableTab, self).__init__(tab_group, request)
if not self.table_classes:
class_name = self.__class__.__name__
raise NotImplementedError("You must define a table_class "
"attribute on %s" % class_name)
# Instantiate our table classes but don't assign data yet
table_instances = [(table._meta.name, table(request,
**tab_group.kwargs))
for table in self.table_classes]
self._tables = SortedDict(table_instances)
self._table_data_loaded = False
def load_table_data(self):
"""
Calls the ``get_{{ table_name }}_data`` methods for each table class
and sets the data on the tables.
"""
# We only want the data to be loaded once, so we track if we have...
if not self._table_data_loaded:
for table_name, table in self._tables.items():
# Fetch the data function.
func_name = "get_%s_data" % table_name
data_func = getattr(self, func_name, None)
if data_func is None:
cls_name = self.__class__.__name__
raise NotImplementedError("You must define a %s method "
"on %s." % (func_name, cls_name))
# Load the data.
table.data = data_func()
table._meta.has_more_data = self.has_more_data(table)
# Mark our data as loaded so we don't run the loaders again.
self._table_data_loaded = True
def get_context_data(self, request):
"""
Adds a ``{{ table_name }}_table`` item to the context for each table
in the :attr:`~horizon.tabs.TableTab.table_classes` attribute.
If only one table class is provided, a shortcut ``table`` context
variable is also added containing the single table.
"""
context = {}
# If the data hasn't been manually loaded before now,
# make certain it's loaded before setting the context.
self.load_table_data()
for table_name, table in self._tables.items():
# If there's only one table class, add a shortcut name as well.
if len(self.table_classes) == 1:
context["table"] = table
context["%s_table" % table_name] = table
return context
def has_more_data(self, table):
return False
def handle(self, *app_labels, **options):
from django.db.models import get_app, get_apps, get_models, get_model
format = options.get('format', 'json')
indent = options.get('indent', None)
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
excludes = options.get('exclude', [])
show_traceback = options.get('traceback', False)
use_natural_keys = options.get('use_natural_keys', False)
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
if not model_obj:
raise CommandError('Unknown model in excludes: %s' %
exclude)
excluded_models.add(model_obj)
else:
try:
app_obj = get_app(exclude)
excluded_apps.add(app_obj)
except ImproperlyConfigured:
raise CommandError('Unknown app in excludes: %s' % exclude)
if len(app_labels) == 0:
app_list = SortedDict(
(app, None) for app in get_apps() if app not in excluded_apps)
else:
app_list = SortedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
if app in excluded_apps:
continue
model = get_model(app_label, model_label)
if model is None:
raise CommandError("Unknown model: %s.%s" %
(app_label, model_label))
if app in app_list.keys():
if app_list[app] and model not in app_list[app]:
app_list[app].append(model)
else:
app_list[app] = [model]
except ValueError:
# This is just an app - no model qualifier
app_label = label
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
if app in excluded_apps:
continue
app_list[app] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Now collate the objects to be serialized.
objects = []
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_syncdb(using, model):
objects.extend(model._default_manager.using(using).all())
try:
return serializers.serialize(format,
objects,
indent=indent,
use_natural_keys=use_natural_keys)
except Exception, e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
class Collector(object):
def __init__(self, using):
self.using = using
# Initially, {model: set([instances])}, later values become lists.
self.data = {}
self.field_updates = {} # {model: {(field, value): set([instances])}}
# fast_deletes is a list of queryset-likes that can be deleted without
# fetching the objects into memory.
self.fast_deletes = []
# Tracks deletion-order dependency for databases without transactions
# or ability to defer constraint checks. Only concrete model classes
# should be included, as the dependencies exist only between actual
# database tables; proxy models are represented here by their concrete
# parent.
self.dependencies = {} # {model: set([models])}
def add(self, objs, source=None, nullable=False, reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted. If the call is
the result of a cascade, 'source' should be the model that caused it,
and 'nullable' should be set to True if the relation can be null.
Returns a list of all objects that were not already collected.
"""
if not objs:
return []
new_objs = []
model = objs[0].__class__
instances = self.data.setdefault(model, set())
for obj in objs:
if obj not in instances:
new_objs.append(obj)
instances.update(new_objs)
# Nullable relationships can be ignored -- they are nulled out before
# deleting, and therefore do not affect the order in which objects have
# to be deleted.
if source is not None and not nullable:
if reverse_dependency:
source, model = model, source
self.dependencies.setdefault(source._meta.concrete_model,
set()).add(model._meta.concrete_model)
return new_objs
def add_field_update(self, field, value, objs):
"""
Schedules a field update. 'objs' must be a homogenous iterable
collection of model instances (e.g. a QuerySet).
"""
if not objs:
return
model = objs[0].__class__
self.field_updates.setdefault(model, {}).setdefault((field, value),
set()).update(objs)
def can_fast_delete(self, objs, from_field=None):
"""
Determines if the objects in the given queryset-like can be
fast-deleted. This can be done if there are no cascades, no
parents and no signal listeners for the object class.
The 'from_field' tells where we are coming from - we need this to
determine if the objects are in fact to be deleted. Allows also
skipping parent -> child -> parent chain preventing fast delete of
the child.
"""
if from_field and from_field.rel.on_delete is not CASCADE:
return False
if not (hasattr(objs, 'model') and hasattr(objs, '_raw_delete')):
return False
model = objs.model
if (signals.pre_delete.has_listeners(model)
or signals.post_delete.has_listeners(model)
or signals.m2m_changed.has_listeners(model)):
return False
# The use of from_field comes from the need to avoid cascade back to
# parent when parent delete is cascading to child.
opts = model._meta
if any(link != from_field
for link in opts.concrete_model._meta.parents.values()):
return False
# Foreign keys pointing to this model, both from m2m and other
# models.
for related in opts.get_all_related_objects(include_hidden=True,
include_proxy_eq=True):
if related.field.rel.on_delete is not DO_NOTHING:
return False
# GFK deletes
for relation in opts.many_to_many:
if not relation.rel.through:
return False
return True
def collect(self,
objs,
source=None,
nullable=False,
collect_related=True,
source_attr=None,
reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted as well as all
parent instances. 'objs' must be a homogenous iterable collection of
model instances (e.g. a QuerySet). If 'collect_related' is True,
related objects will be handled by their respective on_delete handler.
If the call is the result of a cascade, 'source' should be the model
that caused it and 'nullable' should be set to True, if the relation
can be null.
If 'reverse_dependency' is True, 'source' will be deleted before the
current model, rather than after. (Needed for cascading to parent
models, the one case in which the cascade follows the forwards
direction of an FK rather than the reverse direction.)
"""
if self.can_fast_delete(objs):
self.fast_deletes.append(objs)
return
new_objs = self.add(objs,
source,
nullable,
reverse_dependency=reverse_dependency)
if not new_objs:
return
model = new_objs[0].__class__
# Recursively collect concrete model's parent models, but not their
# related objects. These will be found by meta.get_all_related_objects()
concrete_model = model._meta.concrete_model
for ptr in six.itervalues(concrete_model._meta.parents):
if ptr:
# FIXME: This seems to be buggy and execute a query for each
# parent object fetch. We have the parent data in the obj,
# but we don't have a nice way to turn that data into parent
# object instance.
parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
self.collect(parent_objs,
source=model,
source_attr=ptr.rel.related_name,
collect_related=False,
reverse_dependency=True)
if collect_related:
for related in model._meta.get_all_related_objects(
include_hidden=True, include_proxy_eq=True):
field = related.field
if field.rel.on_delete == DO_NOTHING:
continue
sub_objs = self.related_objects(related, new_objs)
if self.can_fast_delete(sub_objs, from_field=field):
self.fast_deletes.append(sub_objs)
elif sub_objs:
field.rel.on_delete(self, field, sub_objs, self.using)
# TODO This entire block is only needed as a special case to
# support cascade-deletes for GenericRelation. It should be
# removed/fixed when the ORM gains a proper abstraction for virtual
# or composite fields, and GFKs are reworked to fit into that.
for relation in model._meta.many_to_many:
if not relation.rel.through:
sub_objs = relation.bulk_related_objects(
new_objs, self.using)
self.collect(sub_objs,
source=model,
source_attr=relation.rel.related_name,
nullable=True)
def related_objects(self, related, objs):
"""
Gets a QuerySet of objects related to ``objs`` via the relation ``related``.
"""
return related.model._base_manager.using(
self.using).filter(**{"%s__in" % related.field.name: objs})
def instances_with_model(self):
for model, instances in six.iteritems(self.data):
for obj in instances:
yield model, obj
def sort(self):
sorted_models = []
concrete_models = set()
models = list(self.data)
while len(sorted_models) < len(models):
found = False
for model in models:
if model in sorted_models:
continue
dependencies = self.dependencies.get(
model._meta.concrete_model)
if not (dependencies
and dependencies.difference(concrete_models)):
sorted_models.append(model)
concrete_models.add(model._meta.concrete_model)
found = True
if not found:
return
self.data = SortedDict([(model, self.data[model])
for model in sorted_models])
@force_managed
def delete(self):
# sort instance collections
for model, instances in self.data.items():
self.data[model] = sorted(instances, key=attrgetter("pk"))
# if possible, bring the models in an order suitable for databases that
# don't support transactions or cannot defer constraint checks until the
# end of a transaction.
self.sort()
# send pre_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.pre_delete.send(sender=model,
instance=obj,
using=self.using)
# fast deletes
for qs in self.fast_deletes:
qs._raw_delete(using=self.using)
# update fields
for model, instances_for_fieldvalues in six.iteritems(
self.field_updates):
query = sql.UpdateQuery(model)
for (field,
value), instances in six.iteritems(instances_for_fieldvalues):
query.update_batch([obj.pk for obj in instances],
{field.name: value}, self.using)
# reverse instance collections
for instances in six.itervalues(self.data):
instances.reverse()
# delete instances
for model, instances in six.iteritems(self.data):
query = sql.DeleteQuery(model)
pk_list = [obj.pk for obj in instances]
query.delete_batch(pk_list, self.using)
# send post_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.post_delete.send(sender=model,
instance=obj,
using=self.using)
# update collected instances
for model, instances_for_fieldvalues in six.iteritems(
self.field_updates):
for (field,
value), instances in six.iteritems(instances_for_fieldvalues):
for obj in instances:
setattr(obj, field.attname, value)
for model, instances in six.iteritems(self.data):
for instance in instances:
setattr(instance, model._meta.pk.attname, None)
def handle_noargs(self, **options):
db = options.get('database')
connection = connections[db]
cursor = connection.cursor()
is_postgres = 'postgres' in settings.DATABASES[db]['ENGINE']
DB_SETTINGS = POSTGRESQL_SETTINGS if is_postgres else MYSQL_SETTINGS
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
# Build the manifest of apps and models that are to be synchronized
all_models = [(app.__name__.split('.')[-2], [
m for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
]) for app in models.get_apps()]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = SortedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
new_models = []
for each in manifest.values():
new_models += each
if len(new_models) > 0:
print colorize('[ERROR] Migration is needed')
print 'Unregistered models:'
for each in manifest.items():
for model in each[1]:
print ' %s' % pretty_name(model)
exit(1)
table_info = []
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
for model in seen_models:
table = model._meta.db_table
columns = [field.column for field in model._meta.fields]
types = {}
nullables = {}
for field in model._meta.fields:
types[field.column] = field.db_type(connection=connection)
nullables[field.column] = field.null
# issue with inheritance
parents = model._meta.parents
for parent in parents:
for field in parent._meta.fields:
columns.remove(field.column)
del types[field.column]
table_info.append((table, columns, types, nullables))
for model in seen_models:
for field in model._meta.local_many_to_many:
if hasattr(field, 'creates_table') and not field.creates_table:
continue
table = field.m2m_db_table()
columns = ['id'] # They always have an id column
types = {}
nullables = {}
types['id'] = 'integer'
nullables['id'] = False
columns.append(field.m2m_column_name())
columns.append(field.m2m_reverse_name())
types[field.m2m_column_name()] = 'integer'
types[field.m2m_reverse_name()] = 'integer'
nullables[field.m2m_column_name()] = False
nullables[field.m2m_reverse_name()] = False
table_info.append((table, columns, types, nullables))
out_of_sync = []
for app_name, model_list in all_models:
for model in model_list:
opts = model._meta
converter = connection.introspection.table_name_converter
table_name = converter(opts.db_table)
for item in table_info:
if item[0] == table_name:
cursor.execute(DB_SETTINGS[TYPES_REQUEST] % table_name)
content = cursor.fetchall()
actual_columns = [column[0] for column in content]
if set(item[1]) != set(actual_columns):
print colorize(
'[ERROR] Model fields are out of sync: %s' %
pretty_name(model),
bold=False)
print ' Model fields: %s' % pretty_list(
item[1])
print ' Database columns: %s' % pretty_list(
actual_columns)
out_of_sync += [model]
break
for djcolname, djcoltype in item[2].items():
dbtypebase = None
for coltype in DB_SETTINGS[MAP]:
if coltype in djcoltype:
dbtypebase = DB_SETTINGS[MAP][coltype]
break
if not dbtypebase:
m = varchar_exp.search(djcoltype)
if m:
dbtypebase = VARCHAR
varchar_length = int(m.group(1))
if not dbtypebase:
print colorize(
'[ERROR] Can\'t validate DB. Unknown field type in %s.%s: %s'
%
(pretty_name(model), djcolname, djcoltype))
exit(1)
for dbcolname, dbcoltype, is_nullable in content:
if dbcolname == djcolname:
if dbtypebase not in dbcoltype:
print colorize(
'[ERROR] Inconsistent field type in model \'%s\''
% pretty_name(model),
bold=False)
print ' Model field: %s %s' % (
djcolname, djcoltype)
print ' Database column: %s %s' % (
dbcolname, dbcoltype)
out_of_sync += [model]
break
if dbtypebase == VARCHAR:
m = varchar_exp.search(dbcoltype)
if not m or varchar_length != int(
m.group(1)):
print colorize(
'[ERROR] Inconsistent varchar length in model \'%s\''
% pretty_name(model),
bold=False)
print ' Model field: %s %d' % (
djcolname, varchar_length)
print ' Database column: %s %d' % (
dbcolname, int(m.group(1)))
out_of_sync += [model]
break
dj_is_nullable = item[3][djcolname]
if dj_is_nullable != yesNoToBool(
is_nullable):
print colorize(
'[ERROR] Inconsistent Nullable in model \'%s\''
% pretty_name(model),
bold=False)
print ' Field: %s' % djcolname
print ' Model NULL: %s (%s)' % (
boolToYesNo(dj_is_nullable),
dj_is_nullable)
print ' Database NULL: %s (%s)' % (
is_nullable,
yesNoToBool(is_nullable))
out_of_sync += [model]
break
break
else:
print colorize(
'[ERROR] Can\'t validate DB. There\'s must be an error in the script!'
)
exit(1)
if out_of_sync:
print colorize('[ERROR] Migration is needed')
print ' Model fields are out of sync:'
for each in set(out_of_sync):
print ' %s' % pretty_name(each)
exit(1)
print colorize('[OK] Migration checks passed', bold=False, green=True)
class UnifiedIndex(object):
# Used to collect all the indexes into a cohesive whole.
def __init__(self, excluded_indexes=None):
self._indexes = {}
self.fields = SortedDict()
self._built = False
self.excluded_indexes = excluded_indexes or []
self.excluded_indexes_ids = {}
self.document_field = getattr(settings, 'HAYSTACK_DOCUMENT_FIELD',
'text')
self._fieldnames = {}
self._facet_fieldnames = {}
@property
def indexes(self):
warnings.warn(
"'UnifiedIndex.indexes' was deprecated in Haystack v2.3.0. Please use UnifiedIndex.get_indexes()."
)
return self._indexes
def collect_indexes(self):
indexes = []
for app_mod in haystack_get_app_modules():
try:
search_index_module = importlib.import_module(
"%s.search_indexes" % app_mod.__name__)
except ImportError:
if module_has_submodule(app_mod, 'search_indexes'):
raise
continue
for item_name, item in inspect.getmembers(search_index_module,
inspect.isclass):
if getattr(item, 'haystack_use_for_indexing',
False) and getattr(item, 'get_model', None):
# We've got an index. Check if we should be ignoring it.
class_path = "%s.search_indexes.%s" % (app_mod.__name__,
item_name)
if class_path in self.excluded_indexes or self.excluded_indexes_ids.get(
item_name) == id(item):
self.excluded_indexes_ids[str(item_name)] = id(item)
continue
indexes.append(item())
return indexes
def reset(self):
self._indexes = {}
self.fields = SortedDict()
self._built = False
self._fieldnames = {}
self._facet_fieldnames = {}
def build(self, indexes=None):
self.reset()
if indexes is None:
indexes = self.collect_indexes()
for index in indexes:
model = index.get_model()
if model in self._indexes:
raise ImproperlyConfigured(
"Model '%s' has more than one 'SearchIndex`` handling it. "
"Please exclude either '%s' or '%s' using the 'EXCLUDED_INDEXES' "
"setting defined in 'settings.HAYSTACK_CONNECTIONS'." %
(model, self._indexes[model], index))
self._indexes[model] = index
self.collect_fields(index)
self._built = True
def collect_fields(self, index):
for fieldname, field_object in index.fields.items():
if field_object.document is True:
if field_object.index_fieldname != self.document_field:
raise SearchFieldError(
"All 'SearchIndex' classes must use the same '%s' fieldname for the 'document=True' field. Offending index is '%s'."
% (self.document_field, index))
# Stow the index_fieldname so we don't have to get it the hard way again.
if fieldname in self._fieldnames and field_object.index_fieldname != self._fieldnames[
fieldname]:
# We've already seen this field in the list. Raise an exception if index_fieldname differs.
raise SearchFieldError(
"All uses of the '%s' field need to use the same 'index_fieldname' attribute."
% fieldname)
self._fieldnames[fieldname] = field_object.index_fieldname
# Stow the facet_fieldname so we don't have to look that up either.
if hasattr(field_object, 'facet_for'):
if field_object.facet_for:
self._facet_fieldnames[field_object.facet_for] = fieldname
else:
self._facet_fieldnames[
field_object.instance_name] = fieldname
# Copy the field in so we've got a unified schema.
if field_object.index_fieldname not in self.fields:
self.fields[field_object.index_fieldname] = field_object
self.fields[field_object.index_fieldname] = copy.copy(
field_object)
else:
# If the field types are different, we can mostly
# safely ignore this. The exception is ``MultiValueField``,
# in which case we'll use it instead, copying over the
# values.
if field_object.is_multivalued:
old_field = self.fields[field_object.index_fieldname]
self.fields[field_object.index_fieldname] = field_object
self.fields[field_object.index_fieldname] = copy.copy(
field_object)
# Switch it so we don't have to dupe the remaining
# checks.
field_object = old_field
# We've already got this field in the list. Ensure that
# what we hand back is a superset of all options that
# affect the schema.
if field_object.indexed is True:
self.fields[field_object.index_fieldname].indexed = True
if field_object.stored is True:
self.fields[field_object.index_fieldname].stored = True
if field_object.faceted is True:
self.fields[field_object.index_fieldname].faceted = True
if field_object.use_template is True:
self.fields[
field_object.index_fieldname].use_template = True
if field_object.null is True:
self.fields[field_object.index_fieldname].null = True
def get_indexes(self):
if not self._built:
self.build()
return self._indexes
def get_indexed_models(self):
# Ensuring a list here since Python3 will give us an iterator
return list(self.get_indexes().keys())
def get_index_fieldname(self, field):
if not self._built:
self.build()
return self._fieldnames.get(field) or field
def get_index(self, model_klass):
indexes = self.get_indexes()
if model_klass not in indexes:
raise NotHandled('The model %s is not registered' % model_klass)
return indexes[model_klass]
def get_facet_fieldname(self, field):
if not self._built:
self.build()
for fieldname, field_object in self.fields.items():
if fieldname != field:
continue
if hasattr(field_object, 'facet_for'):
if field_object.facet_for:
return field_object.facet_for
else:
return field_object.instance_name
else:
return self._facet_fieldnames.get(field) or field
return field
def all_searchfields(self):
if not self._built:
self.build()
return self.fields
class SortedDictTests(SimpleTestCase):
def setUp(self):
self.d1 = SortedDict()
self.d1[7] = 'seven'
self.d1[1] = 'one'
self.d1[9] = 'nine'
self.d2 = SortedDict()
self.d2[1] = 'one'
self.d2[9] = 'nine'
self.d2[0] = 'nil'
self.d2[7] = 'seven'
def test_basic_methods(self):
self.assertEqual(self.d1.keys(), [7, 1, 9])
self.assertEqual(self.d1.values(), ['seven', 'one', 'nine'])
self.assertEqual(self.d1.items(), [(7, 'seven'), (1, 'one'),
(9, 'nine')])
def test_overwrite_ordering(self):
""" Overwriting an item keeps it's place. """
self.d1[1] = 'ONE'
self.assertEqual(self.d1.values(), ['seven', 'ONE', 'nine'])
def test_append_items(self):
""" New items go to the end. """
self.d1[0] = 'nil'
self.assertEqual(self.d1.keys(), [7, 1, 9, 0])
def test_delete_and_insert(self):
"""
Deleting an item, then inserting the same key again will place it
at the end.
"""
del self.d2[7]
self.assertEqual(self.d2.keys(), [1, 9, 0])
self.d2[7] = 'lucky number 7'
self.assertEqual(self.d2.keys(), [1, 9, 0, 7])
def test_change_keys(self):
"""
Changing the keys won't do anything, it's only a copy of the
keys dict.
"""
k = self.d2.keys()
k.remove(9)
self.assertEqual(self.d2.keys(), [1, 9, 0, 7])
def test_init_keys(self):
"""
Initialising a SortedDict with two keys will just take the first one.
A real dict will actually take the second value so we will too, but
we'll keep the ordering from the first key found.
"""
tuples = ((2, 'two'), (1, 'one'), (2, 'second-two'))
d = SortedDict(tuples)
self.assertEqual(d.keys(), [2, 1])
real_dict = dict(tuples)
self.assertEqual(sorted(real_dict.values()), ['one', 'second-two'])
# Here the order of SortedDict values *is* what we are testing
self.assertEqual(d.values(), ['second-two', 'one'])
def test_overwrite(self):
self.d1[1] = 'not one'
self.assertEqual(self.d1[1], 'not one')
self.assertEqual(self.d1.keys(), self.d1.copy().keys())
def test_append(self):
self.d1[13] = 'thirteen'
self.assertEqual(repr(self.d1),
"{7: 'seven', 1: 'one', 9: 'nine', 13: 'thirteen'}")
def test_pop(self):
self.assertEqual(self.d1.pop(1, 'missing'), 'one')
self.assertEqual(self.d1.pop(1, 'missing'), 'missing')
# We don't know which item will be popped in popitem(), so we'll
# just check that the number of keys has decreased.
l = len(self.d1)
self.d1.popitem()
self.assertEqual(l - len(self.d1), 1)
def test_dict_equality(self):
d = SortedDict((i, i) for i in xrange(3))
self.assertEqual(d, {0: 0, 1: 1, 2: 2})
def test_tuple_init(self):
d = SortedDict(((1, "one"), (0, "zero"), (2, "two")))
self.assertEqual(repr(d), "{1: 'one', 0: 'zero', 2: 'two'}")
def test_pickle(self):
self.assertEqual(pickle.loads(pickle.dumps(self.d1, 2)), {
7: 'seven',
1: 'one',
9: 'nine'
})
def test_clear(self):
self.d1.clear()
self.assertEqual(self.d1, {})
self.assertEqual(self.d1.keyOrder, [])
def get_updated_questions_for_user(self, user):
"""
retreive relevant question updates for the user
according to their subscriptions and recorded question
views
"""
user_feeds = EmailFeedSetting.objects.filter(subscriber=user).exclude(
frequency__in=('n', 'i'))
should_proceed = False
for feed in user_feeds:
if feed.should_send_now() == True:
should_proceed = True
break
#shortcircuit - if there is no ripe feed to work on for this user
if should_proceed == False:
return {}
#these are placeholders for separate query sets per question group
#there are four groups - one for each EmailFeedSetting.feed_type
#and each group has subtypes A and B
#that's because of the strange thing commented below
#see note on Q and F objects marked with todo tag
q_sel_A = None
q_sel_B = None
q_ask_A = None
q_ask_B = None
q_ans_A = None
q_ans_B = None
q_all_A = None
q_all_B = None
#base question query set for this user
#basic things - not deleted, not closed, not too old
#not last edited by the same user
base_qs = Post.objects.get_questions(
).exclude(thread__last_activity_by=user).exclude(
thread__last_activity_at__lt=user.date_joined #exclude old stuff
).exclude(deleted=True).exclude(
thread__closed=True).order_by('-thread__last_activity_at')
if askbot_settings.ENABLE_CONTENT_MODERATION:
base_qs = base_qs.filter(approved=True)
#todo: for some reason filter on did not work as expected ~Q(viewed__who=user) |
# Q(viewed__who=user,viewed__when__lt=F('thread__last_activity_at'))
#returns way more questions than you might think it should
#so because of that I've created separate query sets Q_set2 and Q_set3
#plus two separate queries run faster!
#build two two queries based
#questions that are not seen by the user at all
not_seen_qs = base_qs.filter(~Q(viewed__who=user))
#questions that were seen, but before last modification
seen_before_last_mod_qs = base_qs.filter(
Q(viewed__who=user,
viewed__when__lt=F('thread__last_activity_at')))
#shorten variables for convenience
Q_set_A = not_seen_qs
Q_set_B = seen_before_last_mod_qs
if getattr(django_settings, 'ASKBOT_MULTILINGUAL', False):
languages = user.languages.split()
else:
languages = None
for feed in user_feeds:
if feed.feed_type == 'm_and_c':
#alerts on mentions and comments are processed separately
#because comments to questions do not trigger change of last_updated
#this may be changed in the future though, see
#http://askbot.org/en/question/96/
continue
#each group of updates represented by the corresponding
#query set has it's own cutoff time
#that cutoff time is computed for each user individually
#and stored as a parameter "cutoff_time"
#we won't send email for a given question if an email has been
#sent after that cutoff_time
if feed.should_send_now():
if DEBUG_THIS_COMMAND == False:
feed.mark_reported_now()
cutoff_time = feed.get_previous_report_cutoff_time()
if feed.feed_type == 'q_sel':
q_sel_A = Q_set_A.filter(thread__followed_by=user)
q_sel_A.cutoff_time = cutoff_time #store cutoff time per query set
q_sel_B = Q_set_B.filter(thread__followed_by=user)
q_sel_B.cutoff_time = cutoff_time #store cutoff time per query set
elif feed.feed_type == 'q_ask':
q_ask_A = Q_set_A.filter(author=user)
q_ask_A.cutoff_time = cutoff_time
q_ask_B = Q_set_B.filter(author=user)
q_ask_B.cutoff_time = cutoff_time
elif feed.feed_type == 'q_ans':
q_ans_A = Q_set_A.filter(thread__posts__author=user,
thread__posts__post_type='answer')
q_ans_A = q_ans_A[:askbot_settings.MAX_ALERTS_PER_EMAIL]
q_ans_A.cutoff_time = cutoff_time
q_ans_B = Q_set_B.filter(thread__posts__author=user,
thread__posts__post_type='answer')
q_ans_B = q_ans_B[:askbot_settings.MAX_ALERTS_PER_EMAIL]
q_ans_B.cutoff_time = cutoff_time
elif feed.feed_type == 'q_all':
q_all_A = user.get_tag_filtered_questions(Q_set_A)
q_all_B = user.get_tag_filtered_questions(Q_set_B)
q_all_A = q_all_A[:askbot_settings.MAX_ALERTS_PER_EMAIL]
q_all_B = q_all_B[:askbot_settings.MAX_ALERTS_PER_EMAIL]
q_all_A.cutoff_time = cutoff_time
q_all_B.cutoff_time = cutoff_time
#build ordered list questions for the email report
q_list = SortedDict()
#todo: refactor q_list into a separate class?
extend_question_list(q_sel_A, q_list, languages=languages)
extend_question_list(q_sel_B, q_list, languages=languages)
#build list of comment and mention responses here
#it is separate because posts are not marked as changed
#when people add comments
#mention responses could be collected in the loop above, but
#it is inconvenient, because feed_type m_and_c bundles the two
#also we collect metadata for these here
try:
feed = user_feeds.get(feed_type='m_and_c')
if feed.should_send_now():
cutoff_time = feed.get_previous_report_cutoff_time()
comments = Post.objects.get_comments().filter(
added_at__lt=cutoff_time, ).exclude(author=user)
q_commented = list()
for c in comments:
post = c.parent
if post.author != user:
continue
#skip is post was seen by the user after
#the comment posting time
q_commented.append(post.get_origin_post())
extend_question_list(q_commented,
q_list,
cutoff_time=cutoff_time,
add_comment=True,
languages=languages)
mentions = Activity.objects.get_mentions(
mentioned_at__lt=cutoff_time, mentioned_whom=user)
#print 'have %d mentions' % len(mentions)
#MM = Activity.objects.filter(activity_type = const.TYPE_ACTIVITY_MENTION)
#print 'have %d total mentions' % len(MM)
#for m in MM:
# print m
mention_posts = get_all_origin_posts(mentions)
q_mentions_id = [q.id for q in mention_posts]
q_mentions_A = Q_set_A.filter(id__in=q_mentions_id)
q_mentions_A.cutoff_time = cutoff_time
extend_question_list(q_mentions_A,
q_list,
add_mention=True,
languages=languages)
q_mentions_B = Q_set_B.filter(id__in=q_mentions_id)
q_mentions_B.cutoff_time = cutoff_time
extend_question_list(q_mentions_B,
q_list,
add_mention=True,
languages=languages)
except EmailFeedSetting.DoesNotExist:
pass
if user.email_tag_filter_strategy == const.INCLUDE_INTERESTING:
extend_question_list(q_all_A, q_list, languages=languages)
extend_question_list(q_all_B, q_list, languages=languages)
extend_question_list(q_ask_A, q_list, limit=True, languages=languages)
extend_question_list(q_ask_B, q_list, limit=True, languages=languages)
extend_question_list(q_ans_A, q_list, limit=True, languages=languages)
extend_question_list(q_ans_B, q_list, limit=True, languages=languages)
if user.email_tag_filter_strategy == const.EXCLUDE_IGNORED:
extend_question_list(q_all_A,
q_list,
limit=True,
languages=languages)
extend_question_list(q_all_B,
q_list,
limit=True,
languages=languages)
ctype = ContentType.objects.get_for_model(Post)
EMAIL_UPDATE_ACTIVITY = const.TYPE_ACTIVITY_EMAIL_UPDATE_SENT
#up to this point we still don't know if emails about
#collected questions were sent recently
#the next loop examines activity record and decides
#for each question, whether it needs to be included or not
#into the report
for q, meta_data in q_list.items():
#this loop edits meta_data for each question
#so that user will receive counts on new edits new answers, etc
#and marks questions that need to be skipped
#because an email about them was sent recently enough
#also it keeps a record of latest email activity per question per user
try:
#todo: is it possible to use content_object here, instead of
#content type and object_id pair?
update_info = Activity.objects.get(
user=user,
content_type=ctype,
object_id=q.id,
activity_type=EMAIL_UPDATE_ACTIVITY)
emailed_at = update_info.active_at
except Activity.DoesNotExist:
update_info = Activity(user=user,
content_object=q,
activity_type=EMAIL_UPDATE_ACTIVITY)
emailed_at = datetime.datetime(1970, 1, 1) #long time ago
except Activity.MultipleObjectsReturned:
raise Exception(
'server error - multiple question email activities '
'found per user-question pair')
cutoff_time = meta_data[
'cutoff_time'] #cutoff time for the question
#skip question if we need to wait longer because
#the delay before the next email has not yet elapsed
#or if last email was sent after the most recent modification
if emailed_at > cutoff_time or emailed_at > q.thread.last_activity_at:
meta_data['skip'] = True
continue
#collect info on all sorts of news that happened after
#the most recent emailing to the user about this question
q_rev = q.revisions.filter(revised_at__gt=emailed_at)
q_rev = q_rev.exclude(author=user)
#now update all sorts of metadata per question
meta_data['q_rev'] = len(q_rev)
if len(q_rev) > 0 and q.added_at == q_rev[0].revised_at:
meta_data['q_rev'] = 0
meta_data['new_q'] = True
else:
meta_data['new_q'] = False
new_ans = Post.objects.get_answers(user).filter(
thread=q.thread,
added_at__gt=emailed_at,
deleted=False,
)
new_ans = new_ans.exclude(author=user)
meta_data['new_ans'] = len(new_ans)
ans_ids = Post.objects.get_answers(user).filter(
thread=q.thread,
added_at__gt=emailed_at,
deleted=False,
).values_list('id', flat=True)
ans_rev = PostRevision.objects.filter(post__id__in=ans_ids)
ans_rev = ans_rev.exclude(author=user).distinct()
meta_data['ans_rev'] = len(ans_rev)
comments = meta_data.get('comments', 0)
mentions = meta_data.get('mentions', 0)
#print meta_data
#finally skip question if there are no news indeed
if len(q_rev) + len(new_ans) + len(
ans_rev) + comments + mentions == 0:
meta_data['skip'] = True
#print 'skipping'
else:
meta_data['skip'] = False
#print 'not skipping'
update_info.active_at = datetime.datetime.now()
if DEBUG_THIS_COMMAND == False:
update_info.save() #save question email update activity
#q_list is actually an ordered dictionary
#print 'user %s gets %d' % (user.username, len(q_list.keys()))
#todo: sort question list by update time
return q_list
class Collector(object):
def __init__(self, using):
self.using = using
# Initially, {model: set([instances])}, later values become lists.
self.data = {}
self.batches = {} # {model: {field: set([instances])}}
self.field_updates = {} # {model: {(field, value): set([instances])}}
self.dependencies = {} # {model: set([models])}
def add(self, objs, source=None, nullable=False, reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted. If the call is
the result of a cascade, 'source' should be the model that caused it
and 'nullable' should be set to True, if the relation can be null.
Returns a list of all objects that were not already collected.
"""
if not objs:
return []
new_objs = []
model = objs[0].__class__
instances = self.data.setdefault(model, set())
for obj in objs:
if obj not in instances:
new_objs.append(obj)
instances.update(new_objs)
# Nullable relationships can be ignored -- they are nulled out before
# deleting, and therefore do not affect the order in which objects have
# to be deleted.
if new_objs and source is not None and not nullable:
if reverse_dependency:
source, model = model, source
self.dependencies.setdefault(source, set()).add(model)
return new_objs
def add_batch(self, model, field, objs):
"""
Schedules a batch delete. Every instance of 'model' that is related to
an instance of 'obj' through 'field' will be deleted.
"""
self.batches.setdefault(model, {}).setdefault(field,
set()).update(objs)
def add_field_update(self, field, value, objs):
"""
Schedules a field update. 'objs' must be a homogenous iterable
collection of model instances (e.g. a QuerySet).
"""
if not objs:
return
model = objs[0].__class__
self.field_updates.setdefault(model, {}).setdefault((field, value),
set()).update(objs)
def collect(self,
objs,
source=None,
nullable=False,
collect_related=True,
source_attr=None,
reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted as well as all
parent instances. 'objs' must be a homogenous iterable collection of
model instances (e.g. a QuerySet). If 'collect_related' is True,
related objects will be handled by their respective on_delete handler.
If the call is the result of a cascade, 'source' should be the model
that caused it and 'nullable' should be set to True, if the relation
can be null.
If 'reverse_dependency' is True, 'source' will be deleted before the
current model, rather than after. (Needed for cascading to parent
models, the one case in which the cascade follows the forwards
direction of an FK rather than the reverse direction.)
"""
new_objs = self.add(objs,
source,
nullable,
reverse_dependency=reverse_dependency)
if not new_objs:
return
model = new_objs[0].__class__
# Recursively collect parent models, but not their related objects.
# These will be found by meta.get_all_related_objects()
for parent_model, ptr in model._meta.parents.iteritems():
if ptr:
parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
self.collect(parent_objs,
source=model,
source_attr=ptr.rel.related_name,
collect_related=False,
reverse_dependency=True)
if collect_related:
for related in model._meta.get_all_related_objects(
include_hidden=True):
field = related.field
if related.model._meta.auto_created:
self.add_batch(related.model, field, new_objs)
else:
sub_objs = self.related_objects(related, new_objs)
if not sub_objs:
continue
field.rel.on_delete(self, field, sub_objs, self.using)
# TODO This entire block is only needed as a special case to
# support cascade-deletes for GenericRelation. It should be
# removed/fixed when the ORM gains a proper abstraction for virtual
# or composite fields, and GFKs are reworked to fit into that.
for relation in model._meta.many_to_many:
if not relation.rel.through:
sub_objs = relation.bulk_related_objects(
new_objs, self.using)
self.collect(sub_objs,
source=model,
source_attr=relation.rel.related_name,
nullable=True)
def related_objects(self, related, objs):
"""
Gets a QuerySet of objects related to ``objs`` via the relation ``related``.
"""
return related.model._base_manager.using(
self.using).filter(**{"%s__in" % related.field.name: objs})
def instances_with_model(self):
for model, instances in self.data.iteritems():
for obj in instances:
yield model, obj
def sort(self):
sorted_models = []
models = self.data.keys()
while len(sorted_models) < len(models):
found = False
for model in models:
if model in sorted_models:
continue
dependencies = self.dependencies.get(model)
if not (dependencies
and dependencies.difference(sorted_models)):
sorted_models.append(model)
found = True
if not found:
return
self.data = SortedDict([(model, self.data[model])
for model in sorted_models])
@force_managed
def delete(self):
# sort instance collections
for model, instances in self.data.items():
self.data[model] = sorted(instances, key=attrgetter("pk"))
# if possible, bring the models in an order suitable for databases that
# don't support transactions or cannot defer contraint checks until the
# end of a transaction.
self.sort()
# send pre_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.pre_delete.send(sender=model,
instance=obj,
using=self.using)
# update fields
for model, instances_for_fieldvalues in self.field_updates.iteritems():
query = sql.UpdateQuery(model)
for (field,
value), instances in instances_for_fieldvalues.iteritems():
query.update_batch([obj.pk for obj in instances],
{field.name: value}, self.using)
# reverse instance collections
for instances in self.data.itervalues():
instances.reverse()
# delete batches
for model, batches in self.batches.iteritems():
query = sql.DeleteQuery(model)
for field, instances in batches.iteritems():
query.delete_batch([obj.pk for obj in instances], self.using,
field)
# delete instances
for model, instances in self.data.iteritems():
query = sql.DeleteQuery(model)
pk_list = [obj.pk for obj in instances]
query.delete_batch(pk_list, self.using)
# send post_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.post_delete.send(sender=model,
instance=obj,
using=self.using)
# update collected instances
for model, instances_for_fieldvalues in self.field_updates.iteritems():
for (field,
value), instances in instances_for_fieldvalues.iteritems():
for obj in instances:
setattr(obj, field.attname, value)
for model, instances in self.data.iteritems():
for instance in instances:
setattr(instance, model._meta.pk.attname, None)
class UnifiedIndex(object):
# Used to collect all the indexes into a cohesive whole.
def __init__(self, excluded_indexes=None):
self.indexes = {}
self.fields = SortedDict()
self._build_lock = threading.RLock()
self._built = False
self._indexes_setup = False
self.excluded_indexes = excluded_indexes or []
self.excluded_indexes_ids = {}
self.document_field = getattr(settings, 'HAYSTACK_DOCUMENT_FIELD',
'text')
self._fieldnames = {}
self._facet_fieldnames = {}
def collect_indexes(self):
indexes = []
for app in settings.INSTALLED_APPS:
mod = importlib.import_module(app)
try:
search_index_module = importlib.import_module(
"%s.search_indexes" % app)
except ImportError:
if module_has_submodule(mod, 'search_indexes'):
raise
continue
for item_name, item in inspect.getmembers(search_index_module,
inspect.isclass):
if getattr(item, 'haystack_use_for_indexing',
False) and getattr(item, 'get_model', None):
# We've got an index. Check if we should be ignoring it.
class_path = "%s.search_indexes.%s" % (app, item_name)
if class_path in self.excluded_indexes or self.excluded_indexes_ids.get(
item_name) == id(item):
self.excluded_indexes_ids[str(item_name)] = id(item)
continue
indexes.append(item())
return indexes
def reset(self):
self.indexes = {}
self.fields = SortedDict()
self._built = False
self._fieldnames = {}
self._facet_fieldnames = {}
def build(self, indexes=None):
"""
Build/rebuild the index data. This class is not thread-safe when
indexes are passed into this method, because it will trigger reset()
and that clears a bunch of data that other threads might be using.
Luckily, web requests never call this passing in indexes, so there
is no need to lock down access to every piece of data on this class.
Only tests and management commands are not thread-safe, and they don't
use threads.
"""
with self._build_lock:
if not self._built or indexes is not None:
self._build(indexes=indexes)
def _build(self, indexes=None):
self.reset()
if indexes is None:
indexes = self.collect_indexes()
for index in indexes:
model = index.get_model()
if model in self.indexes:
raise ImproperlyConfigured(
"Model '%s' has more than one 'SearchIndex`` handling it. Please exclude either '%s' or '%s' using the 'HAYSTACK_EXCLUDED_INDEXES' setting."
% (model, self.indexes[model], index))
self.indexes[model] = index
self.collect_fields(index)
self._built = True
def collect_fields(self, index):
for fieldname, field_object in index.fields.items():
if field_object.document is True:
if field_object.index_fieldname != self.document_field:
raise SearchFieldError(
"All 'SearchIndex' classes must use the same '%s' fieldname for the 'document=True' field. Offending index is '%s'."
% (self.document_field, index))
# Stow the index_fieldname so we don't have to get it the hard way again.
if fieldname in self._fieldnames and field_object.index_fieldname != self._fieldnames[
fieldname]:
# We've already seen this field in the list. Raise an exception if index_fieldname differs.
raise SearchFieldError(
"All uses of the '%s' field need to use the same 'index_fieldname' attribute."
% fieldname)
self._fieldnames[fieldname] = field_object.index_fieldname
# Stow the facet_fieldname so we don't have to look that up either.
if hasattr(field_object, 'facet_for'):
if field_object.facet_for:
self._facet_fieldnames[field_object.facet_for] = fieldname
else:
self._facet_fieldnames[
field_object.instance_name] = fieldname
# Copy the field in so we've got a unified schema.
if not field_object.index_fieldname in self.fields:
self.fields[field_object.index_fieldname] = field_object
self.fields[field_object.index_fieldname] = copy.copy(
field_object)
else:
# If the field types are different, we can mostly
# safely ignore this. The exception is ``MultiValueField``,
# in which case we'll use it instead, copying over the
# values.
if field_object.is_multivalued == True:
old_field = self.fields[field_object.index_fieldname]
self.fields[field_object.index_fieldname] = field_object
self.fields[field_object.index_fieldname] = copy.copy(
field_object)
# Switch it so we don't have to dupe the remaining
# checks.
field_object = old_field
# We've already got this field in the list. Ensure that
# what we hand back is a superset of all options that
# affect the schema.
if field_object.indexed is True:
self.fields[field_object.index_fieldname].indexed = True
if field_object.stored is True:
self.fields[field_object.index_fieldname].stored = True
if field_object.faceted is True:
self.fields[field_object.index_fieldname].faceted = True
if field_object.use_template is True:
self.fields[
field_object.index_fieldname].use_template = True
if field_object.null is True:
self.fields[field_object.index_fieldname].null = True
def setup_indexes(self):
if not self._built:
self.build()
if self._indexes_setup:
return
for model_ct, index in self.indexes.items():
index._setup_save()
index._setup_delete()
self._indexes_setup = True
def teardown_indexes(self):
if not self._built:
self.build()
for model_ct, index in self.indexes.items():
index._teardown_save()
index._teardown_delete()
self._indexes_setup = False
def get_indexed_models(self):
if not self._built:
self.build()
return self.indexes.keys()
def get_index_fieldname(self, field):
if not self._built:
self.build()
return self._fieldnames.get(field) or field
def get_index(self, model_klass):
if not self._built:
self.build()
if model_klass not in self.indexes:
raise NotHandled('The model %s is not registered' %
model_klass.__class__)
return self.indexes[model_klass]
def get_facet_fieldname(self, field):
if not self._built:
self.build()
for fieldname, field_object in self.fields.items():
if fieldname != field:
continue
if hasattr(field_object, 'facet_for'):
if field_object.facet_for:
return field_object.facet_for
else:
return field_object.instance_name
else:
return self._facet_fieldnames.get(field) or field
return field
def all_searchfields(self):
if not self._built:
self.build()
return self.fields
def handle_inspection(self, options):
connection = connections[options.get('database')]
# 'table_name_filter' is a stealth option
table_name_filter = options.get('table_name_filter')
table2model = lambda table_name: table_name.title().replace(
'_', '').replace(' ', '').replace('-', '')
strip_prefix = lambda s: s[1:] if s.startswith("u'") else s
cursor = connection.cursor()
yield "# This is an auto-generated Django model module."
yield "# You'll have to do the following manually to clean this up:"
yield "# * Rearrange models' order"
yield "# * Make sure each model has one field with primary_key=True"
yield "# * Remove `managed = False` lines if you wish to allow Django to create and delete the table"
yield "# Feel free to rename the models, but don't rename db_table values or field names."
yield "#"
yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
yield "# into your database."
yield "from __future__ import unicode_literals"
yield ''
yield 'from %s import models' % self.db_module
yield ''
known_models = []
for table_name in connection.introspection.table_names(cursor):
if table_name_filter is not None and callable(table_name_filter):
if not table_name_filter(table_name):
continue
yield 'class %s(models.Model):' % table2model(table_name)
known_models.append(table2model(table_name))
try:
relations = connection.introspection.get_relations(
cursor, table_name)
except NotImplementedError:
relations = {}
try:
indexes = connection.introspection.get_indexes(
cursor, table_name)
except NotImplementedError:
indexes = {}
used_column_names = [
] # Holds column names used in the table so far
for i, row in enumerate(
connection.introspection.get_table_description(
cursor, table_name)):
comment_notes = [
] # Holds Field notes, to be displayed in a Python comment.
extra_params = SortedDict(
) # Holds Field parameters such as 'db_column'.
column_name = row[0]
is_relation = i in relations
att_name, params, notes = self.normalize_col_name(
column_name, used_column_names, is_relation)
extra_params.update(params)
comment_notes.extend(notes)
used_column_names.append(att_name)
# Add primary_key and unique, if necessary.
if column_name in indexes:
if indexes[column_name]['primary_key']:
extra_params['primary_key'] = True
elif indexes[column_name]['unique']:
extra_params['unique'] = True
if is_relation:
rel_to = "self" if relations[i][
1] == table_name else table2model(relations[i][1])
if rel_to in known_models:
field_type = 'ForeignKey(%s' % rel_to
else:
field_type = "ForeignKey('%s'" % rel_to
else:
# Calling `get_field_type` to get the field type string and any
# additional paramters and notes.
field_type, field_params, field_notes = self.get_field_type(
connection, table_name, row)
extra_params.update(field_params)
comment_notes.extend(field_notes)
field_type += '('
# Don't output 'id = meta.AutoField(primary_key=True)', because
# that's assumed if it doesn't exist.
if att_name == 'id' and field_type == 'AutoField(' and extra_params == {
'primary_key': True
}:
continue
# Add 'null' and 'blank', if the 'null_ok' flag was present in the
# table description.
if row[6]: # If it's NULL...
if field_type == 'BooleanField(':
field_type = 'NullBooleanField('
else:
extra_params['blank'] = True
if not field_type in ('TextField(', 'CharField('):
extra_params['null'] = True
field_desc = '%s = models.%s' % (att_name, field_type)
if extra_params:
if not field_desc.endswith('('):
field_desc += ', '
field_desc += ', '.join([
'%s=%s' % (k, strip_prefix(repr(v)))
for k, v in extra_params.items()
])
field_desc += ')'
if comment_notes:
field_desc += ' # ' + ' '.join(comment_notes)
yield ' %s' % field_desc
for meta_line in self.get_meta(table_name):
yield meta_line
def handle_noargs(self, **options):
verbosity = int(options.get('verbosity'))
interactive = options.get('interactive')
show_traceback = options.get('traceback')
load_initial_data = options.get('load_initial_data')
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError as exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith(
'No module named') or 'management' not in msg:
raise
db = options.get('database')
connection = connections[db]
cursor = connection.cursor()
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [(app.__name__.split('.')[-2], [
m for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
]) for app in models.get_apps()]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = SortedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
# Create the tables for each model
if verbosity >= 1:
self.stdout.write("Creating tables ...\n")
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if verbosity >= 3:
self.stdout.write("Processing %s.%s model\n" %
(app_name, model._meta.object_name))
sql, references = connection.creation.sql_create_model(
model, self.style, seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, self.style, pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, self.style, pending_references))
if verbosity >= 1 and sql:
self.stdout.write("Creating table %s\n" %
model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
transaction.commit_unless_managed(using=db)
# Send the post_syncdb signal, so individual apps can do whatever they need
# to do at this point.
emit_post_sync_signal(created_models, verbosity, interactive, db)
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
if verbosity >= 1:
self.stdout.write("Installing indexes ...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, self.style)
if index_sql:
if verbosity >= 2:
self.stdout.write(
"Installing index for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
# Install custom SQL for the app (but only if this
# is a model we've just created)
if verbosity >= 1:
self.stdout.write("Installing custom SQL ...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, self.style,
connection)
if custom_sql:
if verbosity >= 2:
self.stdout.write(
"Installing custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
if show_traceback:
traceback.print_exc()
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
else:
if verbosity >= 3:
self.stdout.write(
"No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
# Load initial_data fixtures (unless that has been disabled)
if load_initial_data:
call_command('loaddata',
'initial_data',
verbosity=verbosity,
database=db,
skip_validation=True)
def generate_model(model_description, mapping, db_key=''):
"""Uses instrospection to generate a Django model from a database table.
"""
connection = db.connections[db_key]
cursor = connection.cursor()
table_name = model_description.name
table2model = lambda table_name: table_name.title().replace(
'_', '').replace(' ', '').replace('-', '')
strip_prefix = lambda s: s[1:] if s.startswith("u'") else s
try:
relations = connection.introspection.get_relations(cursor, table_name)
except NotImplementedError:
relations = {}
try:
indexes = connection.introspection.get_indexes(cursor, table_name)
except NotImplementedError:
indexes = {}
used_column_names = [] # Holds column names used in the table so far
for i, row in enumerate(
connection.introspection.get_table_description(cursor,
table_name)):
comment_notes = [
] # Holds Field notes, to be displayed in a Python comment.
extra_params = SortedDict(
) # Holds Field parameters such as 'db_column'.
column_name = row[0]
is_relation = i in relations
att_name, params, notes = normalize_col_name(column_name,
used_column_names,
is_relation)
extra_params.update(params)
comment_notes.extend(notes)
used_column_names.append(att_name)
# Add primary_key and unique, if necessary.
if column_name in indexes:
if indexes[column_name]['primary_key']:
extra_params['primary_key'] = True
elif indexes[column_name]['unique']:
extra_params['unique'] = True
# Calling `get_field_type` to get the field type string and any
# additional parameters and notes
field_type, field_params, field_notes = get_field_type(
connection, table_name, row)
extra_params.update(field_params)
comment_notes.extend(field_notes)
GEOM_FIELDS = {
'GEOMETRYCOLLECTION': 'GeometryCollectionField',
'POINT': 'PointField',
'MULTIPOINT': 'MultiPointField',
'LINESTRING': 'LineStringField',
'MULTILINESTRING': 'MultiLineStringField',
'POLYGON': 'PolygonField',
'MULTIPOLYGON': 'MultiPolygonField',
'GEOMETRY': 'GeometryField',
}
geom_type = mapping['geom']
# Use the geom_type to override the geometry field.
if field_type == 'GeometryField':
if geom_type in GEOM_FIELDS:
field_type = GEOM_FIELDS[geom_type]
# Change the type of id to AutoField to get auto generated ids.
if att_name == 'id' and extra_params == {'primary_key': True}:
field_type = 'AutoField'
# Add 'null' and 'blank', if the 'null_ok' flag was present in the
# table description.
if row[6]: # If it's NULL...
if field_type == 'BooleanField':
field_type = 'NullBooleanField'
else:
extra_params['blank'] = True
if not field_type in ('TextField', 'CharField'):
extra_params['null'] = True
field_desc = (att_name, field_type, extra_params)
if any(field_type) and column_name != 'id':
field, __ = Field.objects.get_or_create(model=model_description,
name=att_name)
field.type = field_type
field.original_name = mapping[column_name]
field.save()
for name, value in extra_params.items():
if any(name):
Setting.objects.get_or_create(field=field,
name=name,
value=value)
def build_changelog(docs_path, package_name="mezzanine"):
"""
Converts Mercurial commits into a changelog in RST format.
"""
project_path = os.path.join(docs_path, "..")
version_file = os.path.join(package_name, "__init__.py")
version_var = "__version__"
changelog_filename = "CHANGELOG"
changelog_file = os.path.join(project_path, changelog_filename)
versions = SortedDict()
repo = None
ignore = ("AUTHORS", "formatting", "typo", "pep8", "pep 8",
"whitespace", "README", "trans", "print debug",
"debugging", "tabs", "style", "sites", "ignore",
"tweak", "cleanup", "minor", "for changeset",
".com``", "oops", "syntax")
hotfixes = {
"40cbc47b8d8a": "1.0.9",
"a25749986abc": "1.0.10",
}
# Load the repo.
try:
from mercurial import ui, hg, error
from mercurial.commands import tag
except ImportError:
pass
else:
try:
ui = ui.ui()
repo = hg.repository(ui, project_path)
except error.RepoError:
return
if repo is None:
return
# Go through each changeset and assign it to the versions dict.
changesets = [repo.changectx(changeset) for changeset in repo.changelog]
for cs in sorted(changesets, reverse=True, key=_changeset_date):
# Check if the file with the version number is in this changeset
# and if it is, pull it out and assign it as a variable.
files = cs.files()
new_version = False
# Commit message cleanup hacks.
description = cs.description().decode("utf-8")
description = description.rstrip(".").replace("\n", ". ")
while " " in description:
description = description.replace(" ", " ")
description = description.replace(". . ", ". ").replace("...", ",")
while ".." in description:
description = description.replace("..", ".")
description = description.replace(":.", ":").replace("n'. t", "n't")
words = description.split()
# Format var names in commit.
for i, word in enumerate(words):
if (set("._") & set(word[:-1]) and set(letters) & set(word) and
"`" not in word and not word[0].isdigit()):
last = ""
if word[-1] in ",.":
last, word = word[-1], word[:-1]
words[i] = "``%s``%s" % (word, last)
description = " ".join(words)
if version_file in files:
for line in cs[version_file].data().split("\n"):
if line.startswith(version_var):
exec(line)
if locals()[version_var] == "0.1.0":
locals()[version_var] = "1.0.0"
break
versions[locals()[version_var]] = {
"changes": [],
"date": _changeset_date(cs).strftime("%b %d, %Y")
}
new_version = len(files) == 1
# Tag new versions.
hotfix = hotfixes.get(cs.hex()[:12])
if hotfix or new_version:
if hotfix:
version_tag = hotfix
else:
try:
version_tag = locals()[version_var]
except KeyError:
version_tag = None
if version_tag and version_tag not in cs.tags():
try:
tag(ui, repo, version_tag, rev=cs.hex())
print("Tagging version %s" % version_tag)
except:
pass
# Ignore changesets that are merges, bumped the version, closed
# a branch, regenerated the changelog itself, contain an ignore
# word, or are one word long.
merge = len(cs.parents()) > 1
branch_closed = len(files) == 0
changelog_update = changelog_filename in files
ignored = [w for w in ignore if w.lower() in description.lower()]
one_word = len(description.split()) == 1
if (merge or new_version or branch_closed or changelog_update or
ignored or one_word):
continue
# Ensure we have a current version and if so, add this changeset's
# description to it.
version = None
try:
version = locals()[version_var]
except KeyError:
if not hotfix:
continue
user = cs.user().decode("utf-8").split("<")[0].strip()
entry = "%s - %s" % (description, user)
if hotfix or entry not in versions[version]["changes"]:
if hotfix:
versions[hotfix] = {
"changes": [entry],
"date": _changeset_date(cs).strftime("%b %d, %Y"),
}
else:
versions[version]["changes"].insert(0, entry)
# Write out the changelog.
with open(changelog_file, "w") as f:
for version, version_info in versions.items():
header = "Version %s (%s)" % (version, version_info["date"])
f.write("%s\n" % header)
f.write("%s\n" % ("-" * len(header)))
f.write("\n")
if version_info["changes"]:
for change in version_info["changes"]:
f.write(" * %s\n" % change)
else:
f.write(" * No changes listed.\n")
f.write("\n")
def rows(self):
# ordered keys with default values
keys = SortedDict([
('catiName', None),
('followedUp', 0),
('noFollowUpAfter6Days', 0),
('waitlisted', 0),
('transferredToTeamLeader', 0),
('timedOut', 0),
('workingDays', 0),
('followUpTime', 0),
('avgTimePerFollowUp', None),
])
db = get_db()
startdate = self.datespan.startdate_param_utc[:10]
enddate = self.datespan.enddate_param_utc[:10]
def get_form_data(user_id):
row = db.view(
'hsph/cati_performance',
startkey=["followUpForm", self.domain, user_id, startdate],
endkey=["followUpForm", self.domain, user_id, enddate],
reduce=True,
wrapper=lambda r: r['value']).first() or {}
if row.get('followUpTime'):
def format(seconds):
return time.strftime('%M:%S', time.gmtime(seconds))
row['avgTimePerFollowUp'] = format(row['followUpTime'] //
row['followUpForms'])
row['followUpTime'] = format(row['followUpTime'])
row['workingDays'] = len(
set(
db.view(
'hsph/cati_performance',
startkey=[
"submissionDay", self.domain, user_id, startdate
],
endkey=[
"submissionDay", self.domain, user_id, enddate
],
reduce=False,
wrapper=lambda r: r['value']['submissionDay']).all()))
return row
def get_case_data(group_id):
row = db.view('hsph/cati_performance',
startkey=["all", self.domain, group_id, startdate],
endkey=["all", self.domain, group_id, enddate],
reduce=True,
wrapper=lambda r: r['value']).first() or {}
# These queries can fail if startdate is less than N days before
# enddate. We just catch and supply a default value.
extra_keys = [
('noFollowUpAfter6Days', 0, 13),
('timedOut', 0, 21),
]
for key in extra_keys:
key, default, days = key
try:
row[key] = db.view(
'hsph/cati_performance',
startkey=[key, self.domain, group_id, startdate],
endkey=[
key, self.domain, group_id,
datestring_minus_days(enddate, days)
],
reduce=True,
wrapper=lambda r: r['value'][key]).first()
except restkit.errors.RequestFailed:
row[key] = default
return row
def sum_dicts(*args):
res = {}
for d in args:
for k, v in d.items():
res[k] = res.get(k, 0) + (v or 0)
return res
rows = []
cati_finder = CATIFinder(self.domain)
for data in cati_finder.get_cati_users_data():
user = data['user']
row = get_form_data(user._id)
row.update(
sum_dicts(*[get_case_data(id) for id in data['group_ids']]))
row['catiName'] = self.table_cell(user.raw_username,
user.username_in_report)
list_row = []
for k, v in keys.items():
val = row.get(k, v)
if val is None:
val = '---'
list_row.append(numeric_cell(val))
rows.append(list_row)
return rows
def get_sql_for_new_models(apps=None, using=DEFAULT_DB_ALIAS):
"""
Unashamedly copied and tweaked from django.core.management.commands.syncdb
"""
connection = connections[using]
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
if apps:
apps = [models.get_app(a) for a in apps]
else:
apps = models.get_apps()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app.__name__.split('.')[-2], [
m
for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(using, m)
])
for app in apps
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
db_table_in = (converter(opts.db_table) in tables)
auto_create_in = (
opts.auto_created and
converter(opts.auto_created._meta.db_table) in tables
)
return not (db_table_in or auto_create_in)
manifest = SortedDict(
(app_name, filter(model_installed, model_list))
for app_name, model_list in all_models
)
statements = []
sql = None
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
sql, references = connection.creation.sql_create_model(
model,
no_style(),
seen_models
)
seen_models.add(model)
created_models.add(model)
statements.append("### New Model: %s.%s" % (
app_name,
str(model).replace("'>", "").split(".")[-1]
))
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto,
no_style(),
pending_references
)
)
sql.extend(
connection.creation.sql_for_pending_references(
model,
no_style(),
pending_references
)
)
statements.extend(sql)
custom_sql = None
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(
model,
no_style(),
connection
)
if custom_sql:
statements.extend(custom_sql)
index_sql = None
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model,
no_style()
)
if index_sql:
statements.extend(index_sql)
return statements
