def select_index_files(available_index_files, ingested_index_files):
""" Split the supplied available index files and already ingested index
files to three separate lists:
- new index files that only need to be ingested
- index files that have updates: this is a list of tuples: (old, new)
- deleted index files
"""
available = SortedDict(
(index[:31], index) for index in available_index_files
)
ingested = SortedDict(
(index[:31], index) for index in ingested_index_files
)
index_files_inserted = [
index for base, index in available.iteritems() if base not in ingested
]
index_files_deleted = [
index for base, index in ingested.iteritems() if base not in available
]
common = (
(ingested[base], index) for base, index in available.iteritems()
if base in ingested
)
index_files_updated = [
(old, new) for old, new in common if old[32:47] < new[32:47]
]
return index_files_inserted, index_files_updated, index_files_deleted
def viewProfile(request):
if not request.user.is_authenticated():
return HttpResponseRedirect('/')
if not request.get_full_path().__contains__('id') and request.user.is_superuser:
return HttpResponseRedirect('/')
if request.get_full_path().__contains__('id') and not request.user.is_superuser:
inputCusId = int(request.get_full_path().split('id=')[-1])
if inputCusId != dbaccess.getCustIdByUserId(request.user.id):
return HttpResponseRedirect('/')
else:
custRow = dbaccess.getCustInfoById(int(request.get_full_path().split('id=')[-1]))
else:
if request.get_full_path().__contains__('id'):
custRow = dbaccess.getCustInfoById(int(request.get_full_path().split('id=')[-1]))
else:
cusId = dbaccess.getCustIdByUserId(request.user.id)
custRow = dbaccess.getCustInfoById(cusId)
custInfo = SortedDict([
('Email', custRow[3]),
('First Name', custRow[1]),
('Last Name', custRow[2]),
('Street Name', custRow[6]),
('Postal Code', custRow[5]),
('Contact Number', custRow[4]),
])
custCredit = SortedDict([
('Serial Number', custRow[8]),
('Expiry Date', custRow[7])
])
return render_to_response('userprofile/viewProfile.html', {
'custInfo': custInfo.iteritems(),
'custCredit': custCredit.iteritems(),
}, context_instance=RequestContext(request))
class Choices(object):
"""
A helper for a django model fields choices parameter.
>>> STATUSES = Choices((
(1, 'DRAFT', 'Draft'),
(2, 'PUBLISHED', 'Published'),
))
#Acts like a choices list
>>> list(STATUSES)
[(1, 'Live'), (2, 'Published')]
#Has Attributes
>>> STATUSES.DRAFT
1
#Can retrieve Label to an attribute/value
>>> STATUSES.label(STATUSES.PUBLISHED)
'Published'
"""
def __init__(self, values):
self.labels = SortedDict()
for index, name, label in values:
setattr(self, name, index)
self.labels[index] = label
def __iter__(self):
return self.labels.iteritems()
def __len__(self):
return len(self.labels)
def label(self, index):
return self.labels[index]
def _build_report_data(self):
data = []
tmp_data = SortedDict()
totals = defaultdict(int)
visits = self.visit_queryset().order_by('visit_date')
for visit in visits:
# ideally this would be done in the database
# but grouping really isn't a thing Django does well
date = visit.visit_date
if date not in tmp_data:
tmp_data[date] = defaultdict(int)
tmp_data[date]['visits'] += 1
page_visits = visit.pagevisit_set.count()
tmp_data[date]['pageviews'] += page_visits
tmp_data[date]['duration'] += visit.duration or 0
if page_visits == 1:
tmp_data[date]['bounces'] += 1
if visit.visitor.visit_set.count() > 1:
tmp_data[date]['returning_visits'] += 1
else:
tmp_data[date]['new_visits'] += 1
for visit_date, visit_data in tmp_data.iteritems():
for key, value in visit_data.items():
totals[key] += value
date_label = utils.format_date(visit_date)
data.append(self._build_row(visit_data, date_label))
if data:
data.append(self._build_row(totals, "Totals"))
return data
def medical_categories(request, meeting_pk=None):
meeting = get_object_or_404(Meeting, pk=meeting_pk)
required_categories = MedicalCategory.objects.filter(submissions__timetable_entries__meeting=meeting).order_by('abbrev')
forms = SortedDict()
for cat in required_categories:
forms[cat] = AssignedMedicalCategoryForm(request.POST or None, meeting=meeting, category=cat, prefix='cat%s' % cat.pk)
if request.method == 'POST':
for cat, form in forms.iteritems():
if form.is_valid():
if form.instance and form.instance.board_member:
# remove all participations for a previous selected board member.
# XXX: this may delete manually entered data. (FMD2)
Participation.objects.filter(medical_category=cat, entry__meeting=meeting).delete()
amc = form.save()
# add participations for all timetable entries with matching categories.
# this assumes that all submissions have already been added to the meeting.
for entry in meeting.timetable_entries.filter(submission__medical_categories=cat).distinct():
Participation.objects.get_or_create(medical_category=cat, entry=entry, user=amc.board_member)
return render(request, 'meetings/timetable/medical_categories.html', {
'meeting': meeting,
'forms': forms,
})
def __init__(self, data_orig, **kargs):
# Remove prefix from parameters
data = SortedDict(parse_qsl(urlparse(data_orig).path, keep_blank_values=True))
data_trim = SortedDict()
for key, value in data.iteritems():
data_trim[re.sub("^wp_", "", key)] = value
super(WebToPayResponseForm, self).__init__(data_trim, **kargs)
def sign_params(params, prefix='turbion.'):
"""Add prefix to additional params and sign them"""
params = SortedDict(
[('%s%s' % (prefix, key), value) for key, value in params.iteritems()]
)
params['%ssignature' % prefix] = _sign(params)
return params
def collect_fields_by_section(self):
""" Return a dict of all fields related to this report by section id.
"""
# map of section id to field dict
fields_by_section = SortedDict()
# section id=0 is the "common" section
# fields attached directly to the Report object are always added
# to the common section
fields_by_section[0] = SortedDict()
if self.fields:
report_fields = {}
for f in self.fields.all():
report_fields[f.keyword] = f
fields_by_section[0].update(report_fields)
# Pull in fields from each section (which may add fields to
# the common as well)
for s in Section.objects.filter(report=self):
for secid, fields in s.collect_fields_by_section().iteritems():
if secid not in fields_by_section:
fields_by_section[secid] = fields
else:
# update fields from fields_by_section so that the
# first definition of a field takes precedence.
# For example, if 'start' is defined at the report
# 'common' level, it's field will be used rather
# than that defined in the section. This is useful
# for custimizing the appearance/label/defaults of
# fields pulled in from tables
fields.update(fields_by_section[secid])
fields_by_section[secid] = fields
# Reorder fields in each section according to the field_order list
new_fields_by_section = {}
for i, fields in fields_by_section.iteritems():
# collect all field names
section_fields = fields_by_section[i]
section_field_names = set(section_fields.keys())
ordered_field_names = SortedDict()
# Iterate over the defined order list, which may not
# address all fields
for name in (self.field_order or []):
if name in section_field_names:
ordered_field_names[name] = section_fields[name]
section_field_names.remove(name)
# Preserve the order of any fields left
for name in section_field_names:
ordered_field_names[name] = section_fields[name]
new_fields_by_section[i] = ordered_field_names
return new_fields_by_section
def __init__(self, data_orig, **kargs):
# Remove prefix from parameters
data = SortedDict(
parse_qsl(urlparse(data_orig).path, keep_blank_values=True))
data_trim = SortedDict()
for key, value in data.iteritems():
data_trim[re.sub('^wp_', '', key)] = value
super(WebToPayResponseForm, self).__init__(data_trim, **kargs)
def collect_fields_by_section(self):
""" Return a dict of all fields related to this report by section id.
"""
# map of section id to field dict
fields_by_section = SortedDict()
# section id=0 is the "common" section
# fields attached directly to the Report object are always added
# to the common section
fields_by_section[0] = SortedDict()
if self.fields:
report_fields = {}
for f in self.fields.all():
report_fields[f.keyword] = f
fields_by_section[0].update(report_fields)
# Pull in fields from each section (which may add fields to
# the common as well)
for s in Section.objects.filter(report=self):
for secid, fields in s.collect_fields_by_section().iteritems():
if secid not in fields_by_section:
fields_by_section[secid] = fields
else:
# update fields from fields_by_section so that the
# first definition of a field takes precedence.
# For example, if 'start' is defined at the report
# 'common' level, it's field will be used rather
# than that defined in the section. This is useful
# for custimizing the appearance/label/defaults of
# fields pulled in from tables
fields.update(fields_by_section[secid])
fields_by_section[secid] = fields
# Reorder fields in each section according to the field_order list
new_fields_by_section = {}
for i, fields in fields_by_section.iteritems():
# collect all field names
section_fields = fields_by_section[i]
section_field_names = set(section_fields.keys())
ordered_field_names = SortedDict()
# Iterate over the defined order list, which may not
# address all fields
for name in (self.field_order or []):
if name in section_field_names:
ordered_field_names[name] = section_fields[name]
section_field_names.remove(name)
# Preserve the order of any fields left
for name in section_field_names:
ordered_field_names[name] = section_fields[name]
new_fields_by_section[i] = ordered_field_names
return new_fields_by_section
def extract_params(params, prefix='turbion.'):
"""Extracts additional request params and checks signature"""
params = SortedDict(
[(name[len(prefix):], value)\
for name, value in params.iteritems() if name.startswith(prefix)]
)
signature = params.pop('signature', '')
if signature != _sign(params):
raise ValueError('Parameters signature doesn\'t match')
return params
def send_pending_create_signals(self):
# Group app_labels together
signals = SortedDict()
for (app_label, model_names) in self.pending_create_signals:
try:
signals[app_label].extend(model_names)
except KeyError:
signals[app_label] = list(model_names)
# Send only one signal per app.
for (app_label, model_names) in signals.iteritems():
self.really_send_create_signal(app_label, list(set(model_names)))
self.pending_create_signals = []
class CompletionProvider(Static):
def _new_(self, attrs, parents):
self.pages = SortedDict()
for name, attr in attrs.iteritems():
self._add_to_class_(name, attr)
def __new__(self, page):
return self.pages[page]()
def get_urls(self):
urls = [url('^'+k+'/$', complete, v.extras) for k,v in self.pages.iteritems()]
# print 'Urls:', ["%s -> %s" % (u.regex, unicode(u.default_args)) for u in urls]
return urls
class APIHandler(object):
"""
An APIHandler object holds registered api resources that could be used in
Piston.
"""
def __init__(self, name=None, app_name='api'):
self._registry = SortedDict() # model_class class -> admin_class instance
self.root_path = None
if name is None:
self.name = 'api'
else:
self.name = name
self.app_name = app_name
def register(self, urlpattern, resource, name=None):
if not name:
if not resource.handler.model:
raise RegisterError('name should not be None')
name = resource.handler.model._meta.object_name
if name in self._registry:
raise AlreadyRegistered('API %s has already been registered' % name)
self._registry[name] = {'urlpattern': urlpattern, 'resource': resource}
def unregister(self, name):
"""
Unregisters the given name.
If an api isn't already registered, this will raise NotRegistered.
"""
if name not in self._registry:
raise NotRegistered('The resource %s is not registered' % name)
del self._registry[name]
def get_urls(self):
from django.conf.urls.defaults import patterns, url
# Admin-site-wide views.
urlpatterns = patterns('')
for name, item in self._registry.iteritems():
urlpatterns += patterns('',
url(item['urlpattern'], item['resource'], name=name)
)
return urlpatterns
def urls(self):
return self.get_urls(), self.app_name, self.name
urls = property(urls)
def get_tabbed_filter_links(self):
"""
Example:
configured in report:
tabbed_filter_fields = (
{
'card': {
'images': {
'default': 'default.png',
'2FF': '2ff-image.img',
},
'types': [
'2FF', '2FF/3FF', '2/3/4FF', '4FF'
]
}
})
will result in (keys are slugified, so 2FF/3FF will be 2ff3ff):
[
'card': [
{'2FF': '2ff-image.png'},
{'2FF/3FF', 'default.png'},
],
]
@return a dict for links with optional images.
"""
report = self
result = SortedDict()
for filter_field in report.tabbed_filter_fields:
options = report.tabbed_filter_fields[filter_field]
if not options or options == {} or 'types' not in options:
raise Exception('Cannot construct tabbed filter fields!')
values = SortedDict()
for field_type in options['types']:
if 'images' in options:
if field_type in options['images']:
values[field_type] = options['images'][field_type]
elif 'default' in options['images'].keys():
values[field_type] = options['images']['default']
else:
values[field_type] = None
else:
values[field_type] = None
result[filter_field] = values.items()
return result.iteritems()
class Choices(object):
"""
Easy declarative "choices" tool::
>>> STATUSES = Choices("Live", "Draft")
# Acts like a choices list:
>>> list(STATUSES)
[(1, 'Live'), (2, 'Draft')]
# Easily convert from code to verbose:
>>> STATUSES.verbose(1)
'Live'
# ... and vice versa:
>>> STATUSES.code("Draft")
2
"""
def __init__(self, *args, **kwargs):
self.code_map = SortedDict()
self.verbose_map = {}
for code, verbose in enumerate(args):
# Enumerate starts from 0, but for convention's sake we'd prefer to
# start choices from 1.
self.code_map[code+1] = verbose
self.verbose_map[verbose] = code+1
for code, verbose in kwargs.items():
self.code_map[code] = verbose
self.verbose_map[verbose] = code
def __iter__(self):
return self.code_map.iteritems()
def __len__(self):
return len(self.code_map)
def code(self, verbose):
"""
Return the code version of the verbose name.
"""
return self.verbose_map[verbose]
def verbose(self, code):
"""
Return the verbose name given the code.
"""
return self.code_map[code]
def create_content_types(feincms_model, content_types_by_region_fn):
# retrieve a mapping of content types for each region
types_by_regions = [(r.key, content_types_by_region_fn(r.key)) for r in feincms_model._feincms_all_regions]
# populate a dict of registration parameters for each type
# e.g. type: (category, [regions])
# registration order matters as we want to control the ordering in
# the admin menu. Hence SortedDict.
types_to_register = SortedDict()
for region, category_types in types_by_regions:
for category, types in category_types:
for type in types:
kwargs = {}
if isinstance(type, (list, tuple)):
kwargs = type[1]
type = type[0]
if type not in types_to_register:
types_to_register[type] = (category, set(), kwargs)
types_to_register[type][1].add(region)
for type, params in types_to_register.iteritems():
option_group, regions, kwargs = params
class_name = None
if hasattr(feincms_model, '_get_content_type_class_name'):
class_name= feincms_model._get_content_type_class_name(type)
new_content_type = feincms_model.create_content_type(
type,
regions=regions,
class_name= class_name,
optgroup=option_group,
**kwargs
)
# FeinCMS does not correctly fake the module appearance,
# and shell_plus becomes subsequently confused.
# -- but we need to be careful if using a class_name which
# might already exist in that module, which can create some
# very confusing bugs...
if not hasattr(sys.modules[feincms_model.__module__],
new_content_type.__name__):
setattr(
sys.modules[feincms_model.__module__],
new_content_type.__name__,
new_content_type
)
def read_alignment_distribution_info(self):
distr_file = "alignTable.txt"
if not self.has_file(distr_file):
return None
distribution = self._from_tsv(distr_file)
filtered_distribution = [row for row in distribution
if sum(int(v) for k, v in row.iteritems() if k and k != 'readLen') > 0]
info = SortedDict()
key_labels = SortedDict((('readLen', 'Read Length[bp]'), ('nread', 'Reads'), ('unalign', 'Unmapped'), ('excluded', 'Excluded'),
('clipped', 'Clipped'), ('err0', 'Perfect'), ('err1', '1 mismatch'), ('err2', '≥2 mismatches')))
for key, label in key_labels.iteritems():
vals = [row[key] for row in filtered_distribution]
info[label] = vals
return info
class ISSite(object):
def __init__(self, name='IS'):
self._registry = SortedDict([(group[0], MenuGroup(group[1]))\
for group in settings.MENU_GROUPS.get(name)])
self.name = name
self.app_name = name
sites[name] = self
def register(self, universal_view_class):
universal_view = universal_view_class(self.name)
if not universal_view.menu_group in self._registry.keys():
raise NoMenuGroup('MENU_GROUPS must contains %s for site %s' %
(universal_view.menu_group, self.name))
self._registry[universal_view.menu_group].views[
universal_view.menu_subgroup] = universal_view
if (hasattr(universal_view, 'model')):
model_label = lower('%s.%s' %
(universal_view.model._meta.app_label,
universal_view.model._meta.object_name))
registered_model_views[model_label] = universal_view
registered_views.append(universal_view)
@property
def urls(self):
return self.get_urls(), self.app_name, self.name
def get_urls(self):
urlpatterns = patterns(
'',
# TODO: environment must exist
url(r'^/?$',
login_required(HomeView.as_view(site_name=self.name)),
name="index"),
url(r'^login/$', LoginView.as_view(), name="login"),
url(r'^logout/$', LogoutView.as_view(), name="logout"),
)
for group_name, menu_group in self._registry.iteritems():
for subgroup_name, persoo_view in menu_group.views.iteritems():
urlpatterns += patterns(
'',
url(r'^%s/%s' % (group_name, subgroup_name),
include(persoo_view.get_urls())))
return urlpatterns
class Simple(resource.Resource):
#isLeaf = True
def __init__(self, *largs, **kwargs):
# Almacenamos los patrenos de las urls en
# un diccionario ordenado
resource.Resource.__init__(self, *largs, **kwargs)
self.patterns = SortedDict()
def getChildWithDefault(self, path, request):
# def getChild(self, name, request):
# El comportamiento por defecto es el de los Resource
res = resource.Resource.getChildWithDefault(self, path, request)
if not isinstance(res, NoResource):
return res
# Si no encontramos el recurso
for uri_regex, entry_point in self.patterns.iteritems():
match = uri_regex.match(request.path)
if match:
log.msg("Match!!!")
break
log.msg(u"*"*40)
return self
def render_GET(self, request):
return "Hello, world! Me encuentro en %r." % (request.prepath,)
def uri_map(self, uri_regex, func, opts = None , **kwargs):
'''
Mapea una URL en una función
'''
if not opts:
opts = {}
if not opts.has_key('method'):
opts['method'] = ('GET', )
regex = re.compile(uri_regex)
self.patterns[regex] = ResourceEntryWrapper(func)
log.msg(pformat(self.patterns))
class ISSite(object):
def __init__(self, name='IS'):
self._registry = SortedDict([(group[0], MenuGroup(group[1]))\
for group in settings.MENU_GROUPS.get(name)])
self.name = name
self.app_name = name
sites[name] = self
def register(self, universal_view_class):
universal_view = universal_view_class(self.name)
if not universal_view.menu_group in self._registry.keys():
raise NoMenuGroup('MENU_GROUPS must contains %s for site %s' % (universal_view.menu_group, self.name))
self._registry[universal_view.menu_group].views[universal_view.menu_subgroup] = universal_view
if (hasattr(universal_view, 'model')):
model_label = lower('%s.%s' % (universal_view.model._meta.app_label, universal_view.model._meta.object_name))
registered_model_views[model_label] = universal_view
registered_views.append(universal_view)
@property
def urls(self):
return self.get_urls(), self.app_name, self.name
def get_urls(self):
urlpatterns = patterns('',
# TODO: environment must exist
url(r'^/?$',
login_required(HomeView.as_view(site_name=self.name)), name="index"),
url(r'^login/$', LoginView.as_view(), name="login"),
url(r'^logout/$', LogoutView.as_view(), name="logout"),
)
for group_name, menu_group in self._registry.iteritems():
for subgroup_name, persoo_view in menu_group.views.iteritems():
urlpatterns += patterns('',
url(r'^%s/%s' % (group_name, subgroup_name),
include(persoo_view.get_urls())
)
)
return urlpatterns
class RestSite(server.Site):
'''
Site que permite mapear urls mediante expresiones regulares además del sistema
de recursos tradicional de Twisted.
'''
def __init__(self, *largs, **kwargs):
server.Site.__init__(self, *largs, **kwargs)
self.url_patterns = SortedDict()
def getResourceFor(self, request):
log.msg("El sitio busca el recurso %s" % request.path)
res = server.Site.getResourceFor(self, request)
if not isinstance(res, NoResource):
return res
for regex, callback in self.url_patterns.iteritems():
norm_path = request.path[1:]
log.msg("Buscando %s" % norm_path)
match = regex.match(norm_path)
if match:
kwargs = match.groupdict()
# Generamos un Resource
class GeneratedResource():
def render(self, request):
return callback(request, **kwargs)
return GeneratedResource()
log.msg(res)
return res
def register_uri(self, uri_regex, callback, **opts):
if uri_regex[0] == '/':
uri_regex = uri_regex[1:]
regex = re.compile(uri_regex)
self.url_patterns[regex] = callback
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources', None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get('followlinks', False)):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
any(fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates.")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = {}
for template, nodes in compressor_nodes.iteritems():
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
extra_context = {}
firstnode = template.nodelist[0]
if isinstance(firstnode, ExtendsNode):
# If this template has a ExtendsNode, we apply our patch to
# generate the necessary context, and then use it for all the
# nodes in it, just in case (we don't know which nodes were
# in a block)
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent, firstnode)
try:
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
except (IOError, TemplateSyntaxError, TemplateDoesNotExist):
# That first node we are trying to render might cause more errors
# that we didn't catch when simply creating a Template instance
# above, so we need to catch that (and ignore it, just like above)
# as well.
if verbosity > 0:
log.write("Caught error when rendering extend node from template %s\n" % template.template_name)
continue
for node in nodes:
context.push()
if extra_context and node._block_name:
context['block'] = context.render_context[BLOCK_CONTEXT_KEY].pop(node._block_name)
if context['block']:
context['block'].context = context
key = get_offline_hexdigest(node.nodelist.render(context))
try:
result = node.render(context, forced=True)
except Exception, e:
raise CommandError("An error occured during rendering %s: "
"%s" % (template.template_name, e))
offline_manifest[key] = result
context.pop()
results.append(result)
count += 1
class CreateBaseForm(forms.ModelForm):
def __init__(self, instance=None, parent=None, *args, **kwargs):
super(CreateBaseForm, self).__init__(instance=instance, *args, **kwargs)
self.external = SortedDict()
for hidden in hidden_fields:
self.fields[hidden].widget = forms.widgets.HiddenInput()
for field in [field for field in self._meta.model._meta.fields if field.name not in to_exclude]:
db_type = field.db_type()
if db_type == 'boolean':
# Specjalna obsługa dla boola, ze względu na potrzebę jasności wyboru, lepiej jeżeli
# wyświetlimy RadioSelect, gdzie user *musi* wybrać coś, niż jeśli CheckBox będzie
# dawałe defaultowego False przy braku wyboru
self.fields[field.name] = forms.TypedChoiceField(coerce=lambda choice: {'True': True, 'False': False}[choice],
choices=(('False', 'Nie'), ('True', 'Tak')),
widget=forms.RadioSelect,
initial=self.fields[field.name].initial,
label=self.fields[field.name].label
)
if can_delete and instance:
self.fields.insert(0, "%s_%d_DELETE" % (instance._meta.object_name.lower(), instance.id), forms.BooleanField(label="Usunąć wpis?", required=False, initial=False))
for related_model in related_models:
self.external[related_model] = []
instances = related_model.model.model_class().objects.filter(**{smart_str(related_model.get_rel_id_field()): related_model.extract_id(self.instance)}).order_by('id')
# In previouse line ordering is super important -- I got some issues with individual elements
# selecting (try-except block below) without it
for i in range(related_model.get_max_count()):
ins = None
if not self.instance.pk is None:
try:
ins = instances[i]
except IndexError:
ins = None
prefix="%s_%d" % (related_model.model.model, i)
empty_permitted = related_model.min_count is None or i >= related_model.min_count
df = createform_factory(related_model.model.model_class(), [], [], use_model_m2m_fields,
excluded_fields=[related_model.get_rel_field_name(), related_model.get_rel_ct_field_name()],
can_delete=True,)(instance=ins, prefix=prefix, empty_permitted=empty_permitted, *args, **kwargs)
self.external[related_model].append(df)
categorical_model_name = 'CategoricalValue' # <---- remove this!!
self.external_m2m = SortedDict()
for related_m2m_model in related_m2m_models:
self.external_m2m[related_m2m_model] = []
choice_manager = related_m2m_model.get_choices_manager()
if choice_manager.model._meta.object_name is categorical_model_name:
choices = related_m2m_model.get_choices()
else:
choices = related_m2m_model.get_choices_by_arguments(self.initial or self.instance.__dict__)
for choice in choices:
ins = None
if not self.instance.pk is None:
try:
ins = related_m2m_model.through.model_class()._default_manager.get(**{smart_str(related_m2m_model.from_field): self.instance,
smart_str(related_m2m_model.to_field): choice})
except related_m2m_model.through.model_class().DoesNotExist:
ins = None
prefix="%s_%d" % (related_m2m_model.get_from_model()._meta.object_name.lower(), choice.id)
empty_permitted = True
form = m2m_form_factory([related_m2m_model.from_field], related_m2m_model.to_field, related_m2m_model.through.model_class(), related_m2m_model)(prefix=prefix, instance=ins, select=choice, empty_permitted=empty_permitted, *args, **kwargs)
self.external_m2m[related_m2m_model].append(form)
self.internal_m2m = SortedDict()
if use_model_m2m_fields:
_handled = [m2m_relation.through for m2m_relation in related_m2m_models]
_get = get_model('contenttypes','contenttype').objects.get_for_model
internal_m2ms = (relation for relation in created_model._meta.many_to_many if _get(relation.rel.through) not in _handled)
for related_m2m_model in internal_m2ms:
self.internal_m2m[related_m2m_model] = []
choice_manager = related_m2m_model.rel.to._default_manager
if choice_manager.model._meta.object_name is categorical_model_name:
to_field_name = related_m2m_model.m2m_reverse_field_name()
to_field = related_m2m_model.rel.through._meta.get_field(to_field_name)
choices = choice_manager.complex_filter(to_field.rel.limit_choices_to)
else:
choices = choice_manager.all()
for choice in choices:
ins = None
if not self.instance.pk is None:
try:
ins = related_m2m_model.rel.through._default_manager.get(**{related_m2m_model.model._meta.object_name.lower(): self.instance,
related_m2m_model.rel.through._meta.object_name.lower(): choice})
except related_m2m_model.rel.through.DoesNotExist:
ins = None
prefix="%s_%d" % (related_m2m_model.model._meta.object_name.lower(), choice.id)
empty_permitted = True
self.internal_m2m[related_m2m_model].append(m2m_form_factory([related_m2m_model.model._meta.object_name.lower()], related_m2m_model.rel.through._meta.object_name.lower(), related_m2m_model.rel.through)(prefix=prefix, instance=ins, select=choice,
empty_permitted=empty_permitted,
*args, **kwargs))
def has_data(self):
return bool(self.cleaned_data) or (not self.has_changed() and not self.instance.pk is None)
def to_update(self):
return self.has_changed() and not self.to_delete() and self.has_data() and not self.instance.pk is None
def to_insert(self):
return self.has_changed() and self.has_data() and self.instance.pk is None
def to_delete(self):
if not self.instance.pk is None:
return self.cleaned_data.get("%s_%d_DELETE" % (self.instance._meta.object_name.lower(), self.instance.id))
return False
def _post_clean(self):
# We override _post_clean method to correctly handle the relations.
# _post_clean is defined in ModelForm and sets self.instance attributes taken from
# self.initial -- we need this behaviour to get the fully polulated instances
# in the child objects
if self._errors:
return
super(CreateBaseForm, self)._post_clean() # this gives us the self.instance update
#for ex in itertools.chain(*self.external.values()): # loops over fk related forms
# setattr(ex.instance, self._meta.model._meta.object_name.lower(), self.instance)
for related_model, related_forms in self.external.iteritems():
for f in related_forms:
# if related_model.rel_type == 'g':
# filter(lambda x: x.name == related_model.rel_field_name, f.instance._meta.virtual_fields)[0]
setattr(f.instance, related_model.rel_field_name, self.instance)
try:
# Check if the proper number of forms is filled
for m2m_model_setup, m2m_forms in itertools.chain(self.external_m2m.iteritems(), self.external.iteritems()):
m2m_count = 0
for f in m2m_forms:
if f.is_valid():
m2m_count += f.to_insert() or f.to_update() or (f.has_data() and not f.to_delete())
if m2m_model_setup.min_count and m2m_count < m2m_model_setup.min_count:
raise forms.ValidationError(u"Wybrano zbyt mało elementów w %s. Minimalna liczba dopuszczalna %d" % (m2m_model_setup.get_model_verbose_name(), m2m_model_setup.min_count))
if m2m_model_setup.max_count and m2m_count > m2m_model_setup.max_count:
raise forms.ValidationError(u"Wybrano zbyt dużo elementów w %s. Maksymalna liczba dopuszczalna %d" % (m2m_model_setup.get_model_verbose_name(), m2m_model_setup.max_count))
except forms.ValidationError, e:
self._errors[NON_FIELD_ERRORS] = self.error_class(e.messages)
def __new__(cls, class_name, bases, attrs):
"""Constructor for a new ModelForm class instance.
The signature of this method is determined by Python internals.
All Django Field instances are removed from attrs and added to
the base_fields attribute instead. Additional Field instances
are added to this based on the Datastore Model class specified
by the Meta attribute.
"""
fields = sorted(((field_name, attrs.pop(field_name))
for field_name, obj in attrs.items()
if isinstance(obj, forms.Field)),
key=lambda obj: obj[1].creation_counter)
for base in bases[::-1]:
if hasattr(base, 'base_fields'):
fields = base.base_fields.items() + fields
declared_fields = SortedDict()
for field_name, obj in fields:
declared_fields[field_name] = obj
opts = ModelFormOptions(attrs.get('Meta', None))
attrs['_meta'] = opts
base_models = []
for base in bases:
base_opts = getattr(base, '_meta', None)
base_model = getattr(base_opts, 'model', None)
if base_model is not None:
base_models.append(base_model)
if len(base_models) > 1:
raise ImproperlyConfigured(
"%s's base classes define more than one model." % class_name)
if opts.model is not None:
if base_models and base_models[0] is not opts.model:
raise ImproperlyConfigured(
'%s defines a different model than its parent.' % class_name)
model_fields = SortedDict()
for name, prop in sorted(opts.model.properties().iteritems(),
key=lambda prop: prop[1].creation_counter):
if opts.fields and name not in opts.fields:
continue
if opts.exclude and name in opts.exclude:
continue
form_field = prop.get_form_field()
if form_field is not None:
model_fields[name] = form_field
if opts.widgets and name in opts.widgets:
model_fields[name].widget = opts.widgets[name]
model_fields.update(declared_fields)
attrs['base_fields'] = model_fields
props = opts.model.properties()
for name, field in model_fields.iteritems():
prop = props.get(name)
if prop:
def clean_for_property_field(value, initial=None, prop=prop,
old_clean=field.clean):
value = old_clean(value)
djangoforms.property_clean(prop, value)
return value
field.clean = clean_for_property_field
else:
attrs['base_fields'] = declared_fields
return super(djangoforms.ModelFormMetaclass, cls).__new__(cls,
class_name, bases, attrs)
class CreateBaseForm(forms.ModelForm):
def __init__(self, instance=None, parent=None, *args, **kwargs):
super(CreateBaseForm, self).__init__(instance=instance, *args, **kwargs)
self.external = SortedDict()
for hidden in hidden_fields:
self.fields[hidden].widget = forms.widgets.HiddenInput()
for field in [field for field in self._meta.model._meta.fields if field.name not in to_exclude]:
db_type = field.db_type()
if db_type == 'boolean':
# Specjalna obsługa dla boola, ze względu na potrzebę jasności wyboru, lepiej jeżeli
# wyświetlimy RadioSelect, gdzie user *musi* wybrać coś, niż jeśli CheckBox będzie
# dawałe defaultowego False przy braku wyboru
self.fields[field.name] = forms.TypedChoiceField(coerce=lambda choice: {'True': True, 'False': False}[choice],
choices=(('False', 'Nie'), ('True', 'Tak')),
widget=forms.RadioSelect,
initial=self.fields[field.name].initial,
label=self.fields[field.name].label
)
if can_delete and instance:
self.fields.insert(0, "%s_%d_DELETE" % (instance._meta.object_name.lower(), instance.id), forms.BooleanField(label="Usunąć wpis?", required=False, initial=False))
for related_model in related_models:
self.external[related_model] = []
instances = related_model.model.model_class().objects.filter(**{smart_str(related_model.get_rel_id_field()): related_model.extract_id(self.instance)}).order_by('id')
# In previouse line ordering is super important -- I got some issues with individual elements
# selecting (try-except block below) without it
for i in range(related_model.get_max_count()):
ins = None
if not self.instance.pk is None:
try:
ins = instances[i]
except IndexError:
ins = None
prefix="%s_%d" % (related_model.model.model, i)
empty_permitted = related_model.min_count is None or i >= related_model.min_count
df = createform_factory(related_model.model.model_class(), [], [], use_model_m2m_fields,
excluded_fields=[related_model.get_rel_field_name(), related_model.get_rel_ct_field_name()],
can_delete=True,)(instance=ins, prefix=prefix, empty_permitted=empty_permitted, *args, **kwargs)
self.external[related_model].append(df)
categorical_model_name = 'CategoricalValue' # <---- remove this!!
self.external_m2m = SortedDict()
for related_m2m_model in related_m2m_models:
self.external_m2m[related_m2m_model] = []
choice_manager = related_m2m_model.get_choices_manager()
if choice_manager.model._meta.object_name is categorical_model_name:
choices = related_m2m_model.get_choices()
else:
choices = related_m2m_model.get_choices_by_arguments(self.initial or self.instance.__dict__)
for choice in choices:
ins = None
if not self.instance.pk is None:
try:
ins = related_m2m_model.through.model_class()._default_manager.get(**{smart_str(related_m2m_model.from_field): self.instance,
smart_str(related_m2m_model.to_field): choice})
except related_m2m_model.through.model_class().DoesNotExist:
ins = None
prefix="%s_%d" % (related_m2m_model.get_from_model()._meta.object_name.lower(), choice.id)
empty_permitted = True
form = m2m_form_factory([related_m2m_model.from_field], related_m2m_model.to_field, related_m2m_model.through.model_class(), related_m2m_model)(prefix=prefix, instance=ins, select=choice, empty_permitted=empty_permitted, *args, **kwargs)
self.external_m2m[related_m2m_model].append(form)
self.internal_m2m = SortedDict()
if use_model_m2m_fields:
_handled = [m2m_relation.through for m2m_relation in related_m2m_models]
_get = get_model('contenttypes','contenttype').objects.get_for_model
internal_m2ms = (relation for relation in created_model._meta.many_to_many if _get(relation.rel.through) not in _handled)
for related_m2m_model in internal_m2ms:
self.internal_m2m[related_m2m_model] = []
choice_manager = related_m2m_model.rel.to._default_manager
if choice_manager.model._meta.object_name is categorical_model_name:
to_field_name = related_m2m_model.m2m_reverse_field_name()
to_field = related_m2m_model.rel.through._meta.get_field(to_field_name)
choices = choice_manager.complex_filter(to_field.rel.limit_choices_to)
else:
choices = choice_manager.all()
for choice in choices:
ins = None
if not self.instance.pk is None:
try:
ins = related_m2m_model.rel.through._default_manager.get(**{related_m2m_model.model._meta.object_name.lower(): self.instance,
related_m2m_model.rel.through._meta.object_name.lower(): choice})
except related_m2m_model.rel.through.DoesNotExist:
ins = None
prefix="%s_%d" % (related_m2m_model.model._meta.object_name.lower(), choice.id)
empty_permitted = True
self.internal_m2m[related_m2m_model].append(m2m_form_factory([related_m2m_model.model._meta.object_name.lower()], related_m2m_model.rel.through._meta.object_name.lower(), related_m2m_model.rel.through)(prefix=prefix, instance=ins, select=choice,
empty_permitted=empty_permitted,
*args, **kwargs))
def has_data(self):
return bool(self.cleaned_data) or (not self.has_changed() and not self.instance.pk is None)
def to_update(self):
return self.has_changed() and not self.to_delete() and self.has_data() and not self.instance.pk is None
def to_insert(self):
return self.has_changed() and self.has_data() and self.instance.pk is None
def to_delete(self):
if not self.instance.pk is None:
return self.cleaned_data.get("%s_%d_DELETE" % (self.instance._meta.object_name.lower(), self.instance.id))
return False
def _post_clean(self):
# We override _post_clean method to correctly handle the relations.
# _post_clean is defined in ModelForm and sets self.instance attributes taken from
# self.initial -- we need this behaviour to get the fully polulated instances
# in the child objects
super(CreateBaseForm, self)._post_clean() # this gives us the self.instance update
#for ex in itertools.chain(*self.external.values()): # loops over fk related forms
# setattr(ex.instance, self._meta.model._meta.object_name.lower(), self.instance)
for related_model, related_forms in self.external.iteritems():
for f in related_forms:
# if related_model.rel_type == 'g':
# filter(lambda x: x.name == related_model.rel_field_name, f.instance._meta.virtual_fields)[0]
setattr(f.instance, related_model.rel_field_name, self.instance)
try:
# Check if the proper number of forms is filled
for m2m_model_setup, m2m_forms in itertools.chain(self.external_m2m.iteritems(), self.external.iteritems()):
m2m_count = 0
for f in m2m_forms:
if f.is_valid():
m2m_count += f.to_insert() or f.to_update() or (f.has_data() and not f.to_delete())
if m2m_model_setup.min_count and m2m_count < m2m_model_setup.min_count:
raise forms.ValidationError(u"Wybrano zbyt mało elementów w %s. Minimalna liczba dopuszczalna %d" % (m2m_model_setup.get_model_verbose_name(), m2m_model_setup.min_count))
if m2m_model_setup.max_count and m2m_count > m2m_model_setup.max_count:
raise forms.ValidationError(u"Wybrano zbyt dużo elementów w %s. Maksymalna liczba dopuszczalna %d" % (m2m_model_setup.get_model_verbose_name(), m2m_model_setup.max_count))
except forms.ValidationError, e:
self._errors[NON_FIELD_ERRORS] = self.error_class(e.messages)
def index(request):
"""View that renders the main search page and results.
"""
context = {}
if request.method == 'POST':
post_data = request.POST
query = post_data.get('query', None)
if query:
meta_object = FetchMetaData.objects.create(
search_key = query, search_stamp=datetime.datetime.now())
meta_object.save()
return redirect('%s?%s' % (reverse('fatninja.views.index'),
urllib.urlencode({'q': query})))
elif request.method == 'GET':
get_data = request.GET
query = get_data.get('q')
if not query:
context['no_header_search_box'] = True
return render_to_response(
'fatninja/home.html', RequestContext(request, context))
context['query'] = query
stripped_query = query.strip()
if stripped_query.startswith('@'):
fetcher = Fetcher()
results = fetcher.userfetch(stripped_query[1:], start_page=1, num_pages=16)
else:
fetcher = Fetcher()
results = fetcher.fetch(stripped_query, start_page=1, num_pages=10)
tweets_to_classify = []
predicted_tweets = SortedDict()
tweets_to_classify_id_map = {}
data_to_db = {}
#for result in results:
for tweet in results:
tweet_id = tweet['id']
cached_tweet = cache.get(tweet_id)
# Check if tweet is cached.
cached_tweet = None
if cached_tweet:
predicted_tweets[tweet_id] = cached_tweet
else:
created_at = datetime.datetime(
*email.utils.parsedate_tz(tweet['created_at'])[:7])
predicted_tweets[tweet_id] = {
'text': tweet['text'],
'date': created_at,
'user': tweet.get('from_user', stripped_query[1:])
}
# If not go check the database
database_tweet = Tweet.objects.with_id(str(tweet_id))
if (database_tweet and
database_tweet.sentiment in [-1, 0, 1]):
predicted_tweets[tweet_id]['sentiment'] = \
database_tweet.sentiment
cache.add(tweet_id, predicted_tweets[tweet_id])
else:
# If it is not even in the database, you are screwed :P
# Go classify it.
tweets_to_classify_id_map[len(tweets_to_classify)] = \
tweet['id']
tweets_to_classify.append(tweet['text'])
tweet['created_at'] = created_at
data_to_db[tweet_id] = tweet
if tweets_to_classify:
classifiers_file = open(os.path.join(datasettings.DATA_DIRECTORY,
'classifiers.pickle'))
classifiers = cPickle.load(classifiers_file)
vectorizer_file = open(os.path.join(datasettings.DATA_DIRECTORY,
'vectorizer.pickle'))
vectorizer = cPickle.load(vectorizer_file)
tweets_vector = vectorizer.transform(tweets_to_classify)
classified = list(classifiers[2].predict(tweets_vector))
else:
classified = []
for pointer, tweet_id in tweets_to_classify_id_map.items():
sentiment = classified[pointer]
predicted_tweets[tweet_id]['sentiment'] = sentiment
# Write it both to the disk and the cache
tweet = data_to_db[tweet_id]
tweet['sentiment'] = sentiment
Tweet(**tweet).save()
cache.add(tweet_id, {
'text': tweet['text'],
'date': tweet['created_at'],
'user': tweet.get('from_user', stripped_query[1:]),
'sentiment': sentiment
})
context['tweets_classified'] = len(predicted_tweets)
context['positive_count'] = 0
context['negative_count'] = 0
context['neutral_count'] = 0
for key, value in predicted_tweets.iteritems():
if value['sentiment'] == 1:
context['positive_count'] += 1
elif value['sentiment'] == 0:
context['neutral_count'] += 1
else:
context['negative_count'] += 1
context['classified_information'] = predicted_tweets
return render_to_response(
'fatninja/home.html', RequestContext(request, context))
class NexusSite(object):
def __init__(self, name=None, app_name='nexus'):
self._registry = {}
self._categories = SortedDict()
if name is None:
self.name = 'nexus'
else:
self.name = name
self.app_name = app_name
def register_category(self, category, label, index=None):
if index:
self._categories.insert(index, category, label)
else:
self._categories[category] = label
def register(self, module, namespace=None, category=None):
module = module(self, category)
if not namespace:
namespace = module.get_namespace()
if namespace:
module.app_name = module.name = namespace
self._registry[namespace] = (module, category)
return module
def unregister(self, namespace):
if namespace in self._registry:
del self._registry[namespace]
def get_urls(self):
from django.conf.urls.defaults import patterns, url, include
base_urls = patterns('',
url(r'^media/(?P<module>[^/]+)/(?P<path>.+)$', self.media, name='media'),
url(r'^$', self.as_view(self.dashboard), name='index'),
url(r'^login/$', self.login, name='login'),
url(r'^logout/$', self.as_view(self.logout), name='logout'),
), self.app_name, self.name
urlpatterns = patterns('',
url(r'^', include(base_urls)),
)
for namespace, module in self.get_modules():
urlpatterns += patterns('',
url(r'^%s/' % namespace, include(module.urls)),
)
return urlpatterns
def urls(self):
return self.get_urls()
urls = property(urls)
def has_permission(self, request, extra_permission=None):
"""
Returns True if the given HttpRequest has permission to view
*at least one* page in the admin site.
For EmployInsight, restricting this to just superusers.
"""
permission = (request.user.is_active and request.user.is_staff
and request.user.is_superuser)
if extra_permission:
permission = permission and request.user.has_perm(extra_permission)
return permission
def as_view(self, view, cacheable=False, extra_permission=None):
"""
Wraps a view in authentication/caching logic
extra_permission can be used to require an extra permission for this view, such as a module permission
"""
def inner(request, *args, **kwargs):
if not self.has_permission(request, extra_permission):
# show login pane
return self.login(request)
return view(request, *args, **kwargs)
# Mark it as never_cache
if not cacheable:
inner = never_cache(inner)
# We add csrf_protect here so this function can be used as a utility
# function for any view, without having to repeat 'csrf_protect'.
if not getattr(view, 'csrf_exempt', False):
inner = csrf_protect(inner)
return update_wrapper(inner, view)
def get_context(self, request):
context = csrf(request)
context.update({
'request': request,
'nexus_site': self,
'nexus_media_prefix': conf.MEDIA_PREFIX.rstrip('/'),
})
return context
def get_modules(self):
for k, v in self._registry.iteritems():
yield k, v[0]
def get_module(self, module):
return self._registry[module][0]
def get_categories(self):
for k, v in self._categories.iteritems():
yield k, v
def get_category_label(self, category):
return self._categories.get(category, category.title().replace('_', ' '))
def render_to_string(self, template, context, request, current_app=None):
if not current_app:
current_app = self.name
else:
current_app = '%s:%s' % (self.name, current_app)
if request:
context_instance = RequestContext(request, current_app=current_app)
else:
context_instance = None
context.update(self.get_context(request))
return render_to_string(template, context,
context_instance=context_instance
)
def render_to_response(self, template, context, request, current_app=None):
"Shortcut for rendering to response and default context instances"
if not current_app:
current_app = self.name
else:
current_app = '%s:%s' % (self.name, current_app)
if request:
context_instance = RequestContext(request, current_app=current_app)
else:
context_instance = None
context.update(self.get_context(request))
return render_to_response(template, context,
context_instance=context_instance
)
## Our views
def media(self, request, module, path):
"""
Serve static files below a given point in the directory structure.
"""
if module == 'nexus':
document_root = os.path.join(NEXUS_ROOT, 'media')
else:
document_root = self.get_module(module).media_root
path = posixpath.normpath(urllib.unquote(path))
path = path.lstrip('/')
newpath = ''
for part in path.split('/'):
if not part:
# Strip empty path components.
continue
drive, part = os.path.splitdrive(part)
head, part = os.path.split(part)
if part in (os.curdir, os.pardir):
# Strip '.' and '..' in path.
continue
newpath = os.path.join(newpath, part).replace('\\', '/')
if newpath and path != newpath:
return HttpResponseRedirect(newpath)
fullpath = os.path.join(document_root, newpath)
if os.path.isdir(fullpath):
raise Http404("Directory indexes are not allowed here.")
if not os.path.exists(fullpath):
raise Http404('"%s" does not exist' % fullpath)
# Respect the If-Modified-Since header.
statobj = os.stat(fullpath)
mimetype = mimetypes.guess_type(fullpath)[0] or 'application/octet-stream'
if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
statobj[stat.ST_MTIME], statobj[stat.ST_SIZE]):
return HttpResponseNotModified(mimetype=mimetype)
contents = open(fullpath, 'rb').read()
response = HttpResponse(contents, mimetype=mimetype)
response["Last-Modified"] = http_date(statobj[stat.ST_MTIME])
response["Content-Length"] = len(contents)
return response
def login(self, request, form_class=None):
"Login form"
from django.contrib.auth import login as login_
from django.contrib.auth.forms import AuthenticationForm
if form_class is None:
form_class = AuthenticationForm
if request.POST:
form = form_class(request, request.POST)
if form.is_valid():
login_(request, form.get_user())
request.session.save()
return HttpResponseRedirect(request.POST.get('next') or reverse('nexus:index', current_app=self.name))
else:
request.session.set_test_cookie()
else:
form = form_class(request)
request.session.set_test_cookie()
return self.render_to_response('nexus/login.html', {
'form': form,
}, request)
login = never_cache(login)
def logout(self, request):
"Logs out user and redirects them to Nexus home"
from django.contrib.auth import logout
logout(request)
return HttpResponseRedirect(reverse('nexus:index', current_app=self.name))
def dashboard(self, request):
"Basic dashboard panel"
# TODO: these should be ajax
module_set = []
for namespace, module in self.get_modules():
home_url = module.get_home_url(request)
if hasattr(module, 'render_on_dashboard'):
# Show by default, unless a permission is required
if not module.permission or request.user.has_perm(module.permission):
module_set.append((module.get_dashboard_title(), module.render_on_dashboard(request), home_url))
return self.render_to_response('nexus/dashboard.html', {
'module_set': module_set,
}, request)
class NexusSite(object):
def __init__(self, name=None, app_name='nexus'):
self._registry = {}
self._categories = SortedDict()
if name is None:
self.name = 'nexus'
else:
self.name = name
self.app_name = app_name
def register_category(self, category, label, index=None):
if index:
self._categories.insert(index, category, label)
else:
self._categories[category] = label
def register(self, module, namespace=None, category=None):
module = module(self, category)
if not namespace:
namespace = module.get_namespace()
if namespace:
module.app_name = module.name = namespace
self._registry[namespace] = (module, category)
return module
def unregister(self, namespace):
if namespace in self._registry:
del self._registry[namespace]
def get_urls(self):
try:
from django.conf.urls import patterns, url, include
except ImportError: # Django <1.4
from django.conf.urls.defaults import patterns, url, include
base_urls = patterns(
'',
url(r'^media/(?P<module>[^/]+)/(?P<path>.+)$',
self.media,
name='media'),
url(r'^$', self.as_view(self.dashboard), name='index'),
url(r'^login/$', self.login, name='login'),
url(r'^logout/$', self.as_view(self.logout), name='logout'),
), self.app_name, self.name
urlpatterns = patterns(
'',
url(r'^', include(base_urls)),
)
for namespace, module in self.get_modules():
urlpatterns += patterns(
'',
url(r'^%s/' % namespace, include(module.urls)),
)
return urlpatterns
def urls(self):
return self.get_urls()
urls = property(urls)
def has_permission(self, request, extra_permission=None):
"""
Returns True if the given HttpRequest has permission to view
*at least one* page in the admin site.
"""
permission = request.user.is_active and request.user.is_staff
if extra_permission:
permission = permission and request.user.has_perm(extra_permission)
return permission
def as_view(self, view, cacheable=False, extra_permission=None):
"""
Wraps a view in authentication/caching logic
extra_permission can be used to require an extra permission for this view, such as a module permission
"""
def inner(request, *args, **kwargs):
if not self.has_permission(request, extra_permission):
# show login pane
return self.login(request)
return view(request, *args, **kwargs)
# Mark it as never_cache
if not cacheable:
inner = never_cache(inner)
# We add csrf_protect here so this function can be used as a utility
# function for any view, without having to repeat 'csrf_protect'.
if not getattr(view, 'csrf_exempt', False):
inner = csrf_protect(inner)
return update_wrapper(inner, view)
def get_context(self, request):
context = csrf(request)
context.update({
'request': request,
'nexus_site': self,
'nexus_media_prefix': conf.MEDIA_PREFIX.rstrip('/'),
})
return context
def get_modules(self):
for k, v in self._registry.iteritems():
yield k, v[0]
def get_module(self, module):
return self._registry[module][0]
def get_categories(self):
for k, v in self._categories.iteritems():
yield k, v
def get_category_label(self, category):
return self._categories.get(category,
category.title().replace('_', ' '))
def render_to_string(self, template, context, request, current_app=None):
if not current_app:
current_app = self.name
else:
current_app = '%s:%s' % (self.name, current_app)
if request:
context_instance = RequestContext(request, current_app=current_app)
else:
context_instance = None
context.update(self.get_context(request))
return render_to_string(template,
context,
context_instance=context_instance)
def render_to_response(self, template, context, request, current_app=None):
"Shortcut for rendering to response and default context instances"
if not current_app:
current_app = self.name
else:
current_app = '%s:%s' % (self.name, current_app)
if request:
context_instance = RequestContext(request, current_app=current_app)
else:
context_instance = None
context.update(self.get_context(request))
return render_to_response(template,
context,
context_instance=context_instance)
## Our views
def media(self, request, module, path):
"""
Serve static files below a given point in the directory structure.
"""
if module == 'nexus':
document_root = os.path.join(NEXUS_ROOT, 'media')
else:
document_root = self.get_module(module).media_root
path = posixpath.normpath(urllib.unquote(path))
path = path.lstrip('/')
newpath = ''
for part in path.split('/'):
if not part:
# Strip empty path components.
continue
drive, part = os.path.splitdrive(part)
head, part = os.path.split(part)
if part in (os.curdir, os.pardir):
# Strip '.' and '..' in path.
continue
newpath = os.path.join(newpath, part).replace('\\', '/')
if newpath and path != newpath:
return HttpResponseRedirect(newpath)
fullpath = os.path.join(document_root, newpath)
if os.path.isdir(fullpath):
raise Http404("Directory indexes are not allowed here.")
if not os.path.exists(fullpath):
raise Http404('"%s" does not exist' % fullpath)
# Respect the If-Modified-Since header.
statobj = os.stat(fullpath)
mimetype = mimetypes.guess_type(
fullpath)[0] or 'application/octet-stream'
if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
statobj[stat.ST_MTIME],
statobj[stat.ST_SIZE]):
return HttpResponseNotModified(content_type=mimetype)
contents = open(fullpath, 'rb').read()
response = HttpResponse(contents, content_type=mimetype)
response["Last-Modified"] = http_date(statobj[stat.ST_MTIME])
response["Content-Length"] = len(contents)
return response
def login(self, request, form_class=None):
"Login form"
from django.contrib.auth import login as login_
from django.contrib.auth.forms import AuthenticationForm
if form_class is None:
form_class = AuthenticationForm
if request.POST:
form = form_class(request, request.POST)
if form.is_valid():
login_(request, form.get_user())
request.session.save()
return HttpResponseRedirect(
request.POST.get('next')
or reverse('nexus:index', current_app=self.name))
else:
request.session.set_test_cookie()
else:
form = form_class(request)
request.session.set_test_cookie()
return self.render_to_response('nexus/login.html', {
'form': form,
}, request)
login = never_cache(login)
def logout(self, request):
"Logs out user and redirects them to Nexus home"
from django.contrib.auth import logout
logout(request)
return HttpResponseRedirect(
reverse('nexus:index', current_app=self.name))
def dashboard(self, request):
"Basic dashboard panel"
# TODO: these should be ajax
module_set = []
for namespace, module in self.get_modules():
home_url = module.get_home_url(request)
if hasattr(module, 'render_on_dashboard'):
# Show by default, unless a permission is required
if not module.permission or request.user.has_perm(
module.permission):
module_set.append(
(module.get_dashboard_title(),
module.render_on_dashboard(request), home_url))
return self.render_to_response('nexus/dashboard.html', {
'module_set': module_set,
}, request)
def get_params(self, flat=False):
"""Определяет порядок полей при выводе в списке реквизитов и выдает правильные названия полей"""
from payments.models import REASONS_TO_WITHDRAWAL
order = {
"bankrur":
[("bank_account", _("Bank account")), ("name", _("Name")),
("tin", _("TIN")), ("bank", _("Bank")),
("credit_card_number", _("Credit card number")),
("correspondent", _("Correspondent")), ("bic", _("BIC")),
("payment_details", _("Payment details"))],
"bankeur": [("bank_account", _("Bank account")),
("name", _("Sender")), ("country", _("Country")),
("address", _("Address")), ("bank", _("Bank")),
("bank_swift", _("Bank's SWIFT code")),
("correspondent", _("Correspondent"))],
"bankusd": [("bank_account", _("Bank account")),
("name", _("Sender")), ("country", _("Country")),
("address", _("Address")), ("bank", _("Bank")),
("bank_swift", _("Bank's SWIFT code")),
("correspondent", _("Correspondent"))],
"bankuah":
[("bank_account", _("Bank account")), ("name", _("Name")),
("tin", _("TIN")), ("bank", _("Bank")),
("credit_card_number", _("Credit card number")),
("correspondent", _("Correspondent")), ("bic", _("BIC")),
("payment_details", _("Payment details"))],
"default": []
}
d = SortedDict()
system = self.payment_system
if not system:
return None
if isinstance(system, basestring):
system = load_payment_system(self.payment_system)
if system.slug in order:
key = system.slug
else:
key = "default"
val = namedtuple("NiceKey", ("key", "value"))
if self.params:
for real_key, display_key in order[key]:
if self.params.get(real_key) in ["", None]:
# пустое значение обозначается длинным дефисом mdash
# mark_safe нужен для корректного отображения —
d[display_key] = val(key=real_key,
value=mark_safe("—"))
else:
if real_key == "correspondent" and system.slug in [
"bankeur", "bankusd"
]:
currency = system.currency
if currency in settings.BANK_ACCOUNTS:
d[display_key] = val(
key=real_key,
value=settings.BANK_ACCOUNTS[currency][int(
self.params[real_key])][0])
elif real_key == "country":
d[display_key] = val(key=real_key,
value=get_country(
self.params[real_key]))
elif real_key == "reason":
d[display_key] = val(
key=real_key,
value=REASONS_TO_WITHDRAWAL[self.params[real_key]])
else:
d[display_key] = val(key=real_key,
value=self.params[real_key])
if flat:
d = SortedDict((key, res.value) for (key, res) in d.iteritems())
return d
def __doc_acu(request, id, mode):
'''
Anon/Create/Update
:param id:int - uuid (anon/create) or doc id (update)
:param mode:int (0: anon (print), 1: create, 2: update)
'''
__log_request(request)
if (mode == 2):
item = models.Doc.objects.get(pk=id) # Update only
uuid = item.type
if (uuid not in moduledict):
return 'Template not found'
else:
uuid = id
if (request.method == 'POST') and (mode > 0) and (request.POST.get('_action', None) in set(('print', 'view'))):
mode = 0
tpl = moduledict[uuid]
# 1. check <pkg>.ANON/CREATE/UPDATE
self_func = [K_T_F_ANON, K_T_F_ADD, K_T_F_EDIT][mode]
if (self_func in tpl[K_V_MODULE].__dict__):
return tpl[K_V_MODULE].__dict__[self_func](request, id)
# else:
# 2. get FORM and FORMSETS
formclass = tpl[K_T_FORM]
formsetsclass = tpl[K_T_FORMSETS] # SortedDict of dicts
if request.method == 'POST':
#pprint.pprint(request.POST['_action'])
form = formclass(request.POST)
if (mode == 0): # ANON, Create/Update -> view/print
del form.fields[K_T_F_NAME]
formlist = SortedDict()
isvalid = form.is_valid()
for k, formset in formsetsclass.iteritems():
formlist[k] = formset(request.POST, prefix=k)
isvalid = isvalid and formlist[k].is_valid()
if isvalid:
data = form.cleaned_data
# inject formsets into data
for k, v in formlist.iteritems():
dataset = list()
for i in v.cleaned_data: # list of dicts
if i: # reject empty dicts
dataset.append(i)
if dataset: # reject empty lists
data[k] = dataset # inject datasets into data
__try_to_call(tpl, K_T_F_POST_FORM, data)
# split
if (mode == 0): # ANON > PRINT, C/U -> V/P
if ((K_T_T in tpl[K_V_MODULE].DATA) and (K_T_T_PRINT in tpl[K_V_MODULE].DATA[K_T_T])):
context_dict = {'data': data}
template = tpl[K_V_MODULE].DATA[K_T_T][K_T_T_PRINT]
if (request.POST.get('_action', None) == u'view'):
__try_to_call(tpl, K_T_F_PRE_VIEW, data) # Create/Update -> View
return converter.html2html(request, context_dict, template)
else: # Anon/Create/Update -> PRINT
__try_to_call(tpl, K_T_F_PRE_PRINT, data)
return __doc_print(request, context_dict, template)
else: # tmp dummy
return redirect('dox.views.index')
else: # CREATE/UPDATE -> SAVE
if (mode == 1): # CREATE
name = data[K_T_F_NAME]
else:
item.name = data[K_T_F_NAME]
del data[K_T_F_NAME]
# convert dates
if (K_V_DATES in tpl):
for k in tpl[K_V_DATES]:
utils.date2str(data, k)
__try_to_call(tpl, K_T_F_PRE_SAVE, data)
if (mode == 1): # CREATE
# user, type, name, data
item = models.Doc(user=request.user, type=uuid, name=name, data=json.dumps(data, indent=1, ensure_ascii=False))
else:
item.data = json.dumps(data, indent=1, ensure_ascii=False)
item.save()
return redirect(doc_r, id=item.pk)
else: # GET
if (mode < 2): # ANON, CREATE
form = formclass()
if (mode == 0): # ANON
del form.fields[K_T_F_NAME]
formlist = SortedDict()
for k, formset in formsetsclass.iteritems():
formlist[k] = formset(prefix=k)
else: # UPDATE
data = json.loads(item.data)
data[K_T_F_NAME] = item.name # inject name
# restore dates after loading
if (K_V_DATES in tpl):
for k in tpl[K_V_DATES]:
utils.str2date(data, k)
#pprint.pprint(data)
__try_to_call(tpl, K_T_F_POST_LOAD, data)
#pprint.pprint(data)
__try_to_call(tpl, K_T_F_PRE_FORM, data)
# split form and formsets
# 1. eject formsets
formlist = SortedDict([])
for pfx, formset in formsetsclass.iteritems(): # formsetsclass == SortedDict {name: FormSetClass}
formset_data = dict()
for i, l in enumerate(data.get(pfx, list())): # l:str - formset name; l:
for k, v in l.iteritems():
formset_data[pfx+'-'+str(i)+'-'+k] = v
formset_data.update({
pfx+'-TOTAL_FORMS': len(data[pfx]) if pfx in data else 1,
pfx+'-INITIAL_FORMS': u'0',
pfx+'-MAX_NUM_FORMS': u'',
})
formlist[pfx] = formset(formset_data, prefix=pfx)
if (pfx in data):
del data[pfx]
# 2. else
form = formclass(data)
return render_to_response(
tpl[K_V_MODULE].DATA[K_T_T][K_T_T_FORM] if ((K_T_T in tpl[K_V_MODULE].DATA) and (K_T_T_FORM in tpl[K_V_MODULE].DATA[K_T_T])) else 'auto_form.html',
context_instance=RequestContext(request, {
'name': tpl[K_V_MODULE].DATA[K_T_NAME],
'comments': tpl[K_V_MODULE].DATA[K_T_COMMENTS],
'legend': tpl[K_V_MODULE].DATA.get(K_T_LEGEND, ''),
'uuid': tpl[K_V_MODULE].DATA[K_T_UUID],
'form': form,
'formlist': formlist,
'example': tpl[K_V_MODULE].DATA.get('example', None),
}))
class BaseResource(GlobalSiteMixin, VirtualEndpoint):
'''
A collection of endpoints representing a particular service
'''
resource_class = '' #hint to the client how this resource is used
form_class = EmptyForm
resource_item_class = ResourceItem
name_suffix = 'resource'
resource_adaptor = None
'''The object representing the resource connection. Typically passed in during construction'''
def __init__(self, **kwargs):
assert 'resource_adaptor' in kwargs
self._installed_endpoints = SortedDict()
super(BaseResource, self).__init__(**kwargs)
def fork(self, **kwargs):
kwargs.setdefault('_installed_endpoints', self._installed_endpoints)
return super(BaseResource, self).fork(**kwargs)
@property
def resource(self):
#endpoints have a resource attribute
return self
def post_register(self):
self.register_endpoints()
super(BaseResource, self).post_register()
def get_app_name(self):
"""
Return the application name of this resource.
Provides the return value of the `app_name` property.
"""
return getattr(self, '_app_name', None)
def _get_app_name(self):
return self.get_app_name()
def _set_app_name(self, name):
self._app_name = name
app_name = property(_get_app_name, _set_app_name, None,
'Set or get the application name')
def get_resource_name(self):
"""
Return the name of this resource.
Provides the return value of the `resource_name` property.
"""
return self._resource_name
def _get_resource_name(self):
return self.get_resource_name()
def _set_resource_name(self, name):
self._resource_name = name
resource_name = property(_get_resource_name, _set_resource_name, None,
'Set or get the name of the resource')
def get_resource_slug(self):
"""
Return the slug of this resource.
Provides the return value of the `resource_slug` property.
"""
if hasattr(self, '_resource_slug'):
return self._resource_slug
return slugify(self.get_resource_name())
def _get_resource_slug(self):
return self.get_resource_slug()
def _set_resource_slug(self, slug):
self._resource_slug = slug
resource_slug = property(_get_resource_slug, _set_resource_slug, None,
'Set or get the slug of the resource')
def get_prompt(self):
return self.resource_name
def get_base_url_name_suffix(self):
if self.base_url_name_suffix is None:
return self.resource_slug
return self.base_url_name_suffix
def register_endpoints(self):
self.endpoints = SortedDict()
for endpoint_cls, kwargs in self.get_view_endpoints():
self._register_endpoint(endpoint_cls, **kwargs)
for key, endpoint in self._installed_endpoints.iteritems():
self.endpoints[key] = self.fork(**self.get_endpoint_kwargs())
def register_endpoint(self, endpoint_cls, **kwargs):
endpoint = self._register_endpoint(endpoint_cls, **kwargs)
self._installed_endpoints[endpoint.get_name_suffix()] = endpoint
return endpoint
def _register_endpoint(self, endpoint_cls, **kwargs):
kwargs = self.get_endpoint_kwargs(**kwargs)
endpoint = endpoint_cls(**kwargs)
self.endpoints[endpoint.get_name_suffix()] = endpoint
return endpoint
def get_view_endpoints(self):
"""
Returns a list of tuples containing
(endpoint class, endpoint kwargs)
"""
return []
def get_children_endpoints(self):
return self.endpoints.values()
def reverse(self, name, *args, **kwargs):
return self.site.reverse(name, *args, **kwargs)
def get_state_data(self):
data = super(BaseResource, self).get_state_data()
data.update({
'resource_name': self.resource_name,
'app_name': self.app_name,
})
return data
def get_indexes(self):
return {}
def get_index(self, name):
return self.get_indexes()[name]
def get_index_query(self, name):
raise NotImplementedError
def get_item_url(self, item):
return None
def get_related_resource_from_field(self, field):
return self.site.get_related_resource_from_field(field)
def get_html_type_from_field(self, field):
return self.site.get_html_type_from_field(field)
def get_absolute_url(self):
return self.get_url()
def get_url(self, **kwargs):
return self.get_main_link_prototype().get_url(**kwargs)
def get_resource_link_item(self):
return None
def get_index_endpoint(self):
return self.endpoints['list']
def get_breadcrumb(self):
bread = self.create_link_collection()
bread.add_link(self.get_index_endpoint(),
rel='breadcrumb',
link_factor='LO',
prompt=self.get_prompt())
return bread
def get_breadcrumbs(self):
if self.parent:
breadcrumbs = self.parent.get_breadcrumbs()
else:
breadcrumbs = self.create_link_collection()
breadcrumbs.extend(self.get_breadcrumb())
return breadcrumbs
def get_paginator_kwargs(self):
return {}
def emit_event(self, event, item_list=None):
"""
Fires of the `resource_event` signal
"""
sender = '%s!%s' % (self.get_url_name(), event)
if item_list is None:
item_list = self.get_resource_items()
resource_event.send(sender=sender,
resource=self,
event=event,
item_list=item_list)
def __new__(cls, class_name, bases, attrs):
"""Constructor for a new ModelForm class instance.
The signature of this method is determined by Python internals.
All Django Field instances are removed from attrs and added to
the base_fields attribute instead. Additional Field instances
are added to this based on the Datastore Model class specified
by the Meta attribute.
"""
fields = sorted(((field_name, attrs.pop(field_name))
for field_name, obj in attrs.items()
if isinstance(obj, forms.Field)),
key=lambda obj: obj[1].creation_counter)
for base in bases[::-1]:
if hasattr(base, 'base_fields'):
fields = base.base_fields.items() + fields
declared_fields = SortedDict()
for field_name, obj in fields:
declared_fields[field_name] = obj
opts = ModelFormOptions(attrs.get('Meta', None))
attrs['_meta'] = opts
base_models = []
for base in bases:
base_opts = getattr(base, '_meta', None)
base_model = getattr(base_opts, 'model', None)
if base_model is not None:
base_models.append(base_model)
if len(base_models) > 1:
raise ImproperlyConfigured(
"%s's base classes define more than one model." % class_name)
if opts.model is not None:
if base_models and base_models[0] is not opts.model:
raise ImproperlyConfigured(
'%s defines a different model than its parent.' % class_name)
model_fields = SortedDict()
for name, prop in sorted(opts.model.properties().iteritems(),
key=lambda prop: prop[1].creation_counter):
if opts.fields and name not in opts.fields:
continue
if opts.exclude and name in opts.exclude:
continue
form_field = prop.get_form_field()
if form_field is not None:
model_fields[name] = form_field
if opts.widgets and name in opts.widgets:
model_fields[name].widget = opts.widgets[name]
model_fields.update(declared_fields)
attrs['base_fields'] = model_fields
props = opts.model.properties()
for name, field in model_fields.iteritems():
prop = props.get(name)
if prop:
def clean_for_property_field(value, initial=None, prop=prop,
old_clean=field.clean):
value = old_clean(value)
djangoforms.property_clean(prop, value)
return value
field.clean = clean_for_property_field
else:
attrs['base_fields'] = declared_fields
# We're intentionally not calling our super's __new__ method, but we _do_
# want call the __new__ method on its super class (which is type).
# pylint: disable=bad-super-call
return super(djangoforms.ModelFormMetaclass, cls).__new__(cls,
class_name, bases, attrs)
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module, 'get_template_sources',
None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"http://django.me/template-loaders "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in walk(path,
followlinks=options.get(
'followlinks', False)):
templates.update(
os.path.join(root, name) for name in files
if not name.startswith('.') and any(
fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
compressor_nodes = SortedDict()
for template_name in templates:
try:
template_file = open(template_name)
try:
template = Template(template_file.read().decode(
settings.FILE_CHARSET))
finally:
template_file.close()
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
nodes = list(self.walk_nodes(template))
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates.")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
log.write("Compressing... ")
count = 0
results = []
offline_manifest = {}
for template, nodes in compressor_nodes.iteritems():
context = Context(settings.COMPRESS_OFFLINE_CONTEXT)
extra_context = {}
firstnode = template.nodelist[0]
if isinstance(firstnode, ExtendsNode):
# If this template has a ExtendsNode, we apply our patch to
# generate the necessary context, and then use it for all the
# nodes in it, just in case (we don't know which nodes were
# in a block)
firstnode._old_get_parent = firstnode.get_parent
firstnode.get_parent = MethodType(patched_get_parent,
firstnode)
extra_context = firstnode.render(context)
context.render_context = extra_context.render_context
for node in nodes:
context.push()
if extra_context and node._block_name:
context['block'] = context.render_context[
BLOCK_CONTEXT_KEY].pop(node._block_name)
if context['block']:
context['block'].context = context
key = get_offline_hexdigest(node.nodelist)
try:
result = node.render(context, forced=True)
except Exception, e:
raise CommandError("An error occured during rendering: "
"%s" % e)
offline_manifest[key] = result
context.pop()
results.append(result)
count += 1
class Choices(object):
"""
Easy declarative "choices" tool::
>>> STATUSES = Choices(
... (1, 'live', 'Live'),
... (2, 'draft', 'Draft'),
... (3, 'hidden', 'Not Live'),
... )
# Acts like a choices list:
>>> list(STATUSES)
[(1, 'Live'), (2, 'Draft')]
# Easily convert from code to verbose:
>>> STATUSES.verbose(1)
'Live'
>>> STATUS.prop(1)
'live'
# ... and vice versa:
>>> STATUSES.code("draft")
2
status = models.SmallIntegerField(choices=STATUSES,
... default=STATUSES["live"])
"""
def __init__(self, *args, **kwargs):
self.code_map = SortedDict()
self.prop_map = SortedDict()
self.reverse_map = {}
for code, prop, verbose in args:
self.code_map[code] = verbose
self.prop_map[code] = prop
self.reverse_map[prop] = code
def __iter__(self):
return self.code_map.iteritems()
def __len__(self):
return len(self.code_map)
def __getitem__(self, prop):
return self.code(prop)
def __contains__(self, prop):
return prop in self.reverse_map
def code(self, prop):
"""
Return the code version of the verbose name.
"""
return self.reverse_map[prop]
def prop(self, code):
return self.prop_map[code]
def verbose(self, code):
"""
Return the verbose name given the code.
"""
return self.code_map[code]
def __repr__(self):
return repr(tuple(self.code_map.items()))
def before(self, api, kwargs=None, **extra):
"""Formats given keyword arguments in MetaTrader format.
<command>-arg1=val1|arg2=val2|...
"""
# Thanks to the bug in NEWACCOUNT, we are forced to use
# SortedDict.
def ensure_unicode(string):
if isinstance(string, str):
return string.decode('cp1251')
return unicode(string)
kwargs = SortedDict(kwargs or {})
kwargs.update(extra)
key = kwargs.pop("key", None)
if self.encoded and not key:
raise MT4Error("Key required to call %r" % self)
qs = u"%s-%s" % (self.command.upper(),
u"|".join(u"{}={}".format(*map(ensure_unicode, pair)) for pair in kwargs.iteritems()))
return (qs, ), {"key": key, "sensitive_fields_regexp": getattr(self, "sensitive_fields_regexp", None)}
def generate_plot(self):
# reset the series
self.hcoptions['series'] = []
dss = self.datasource.series
# find all x's from different datasources that need to be plotted on
# same xAxis and also find their corresponding y's
cht_typ_grp = lambda y_term: ('scatter' if
self.series_options[y_term]['type']
in ['scatter', 'pie'] else 'line')
for x_axis_num, vqs_groups in self.x_axis_vqs_groups.items():
y_hco_list = []
try:
x_sortf, x_mapf, x_mts = self.x_sortf_mapf_mts[x_axis_num]
except IndexError:
x_sortf, x_mapf, x_mts = (None, None, False)
ptype_x_y_terms = defaultdict(list)
for vqs_group in vqs_groups.values():
x_term, y_terms_all = vqs_group.items()[0]
y_terms_by_type = defaultdict(list)
for y_term in y_terms_all:
y_terms_by_type[cht_typ_grp(y_term)].append(y_term)
for y_type, y_term_list in y_terms_by_type.items():
ptype_x_y_terms[y_type].append((x_term, y_term_list))
# ptype = plot type i.e. 'line', 'scatter', 'area', etc.
for ptype, x_y_terms_tuples in ptype_x_y_terms.items():
y_fields_multi = []
y_aliases_multi = []
y_types_multi = []
y_hco_list_multi = []
y_values_multi = SortedDict()
y_terms_multi = []
for x_term, y_terms in x_y_terms_tuples:
# x related
x_vqs = dss[x_term]['_data']
x_field = dss[x_term]['field']
# y related
y_fields = [dss[y_term]['field'] for y_term in y_terms]
y_aliases = [dss[y_term]['field_alias'] for y_term
in y_terms]
y_types = [self.series_options[y_term].get('type', 'line')
for y_term in y_terms]
y_hco_list = [HCOptions(
copy.deepcopy(self.series_options[y_term]))
for y_term in y_terms]
for opts, alias, typ in zip(y_hco_list,
y_aliases,
y_types):
opts.pop('_x_axis_term')
opts['name'] = alias
opts['type'] = typ
opts['data'] = []
if ptype == 'scatter' or (ptype == 'line' and
len(x_y_terms_tuples) == 1):
if x_mts:
if x_mapf:
data = ((x_mapf(value_dict[x_field]),
[value_dict[y_field] for y_field
in y_fields])
for value_dict in x_vqs)
sort_key = ((lambda(x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(data, key=sort_key)
else:
sort_key = ((lambda(x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(
((value_dict[x_field],
[value_dict[y_field] for y_field in
y_fields])
for value_dict in x_vqs),
key=sort_key)
if x_mapf:
data = [(x_mapf(x), y) for (x, y) in data]
if ptype == 'scatter':
if self.series_options[y_term]['type'] == 'scatter':
# scatter plot
for x_value, y_value_tuple in data:
for opts, y_value in izip(y_hco_list,
y_value_tuple):
opts['data'].append((x_value, y_value))
self.hcoptions['series'].extend(y_hco_list)
else:
# pie chart
for x_value, y_value_tuple in data:
for opts, y_value in izip(y_hco_list,
y_value_tuple):
opts['data'].append((str(x_value),
y_value))
self.hcoptions['series'].extend(y_hco_list)
if ptype == 'line' and len(x_y_terms_tuples) == 1:
# all other chart types - line, area, etc.
hco_x_axis = self.hcoptions['xAxis']
if len(hco_x_axis) - 1 < x_axis_num:
hco_x_axis.extend([HCOptions({})] *
(x_axis_num -
(len(hco_x_axis) -
1)))
hco_x_axis[x_axis_num]['categories'] = []
for x_value, y_value_tuple in data:
hco_x_axis[x_axis_num]['categories']\
.append(x_value)
for opts, y_value in izip(y_hco_list,
y_value_tuple):
opts['data'].append(y_value)
self.hcoptions['series'].extend(y_hco_list)
else:
data = ((value_dict[x_field],
[value_dict[y_field] for y_field in
y_fields])
for value_dict in x_vqs)
y_terms_multi.extend(y_terms)
y_fields_multi.extend(y_fields)
y_aliases_multi.extend(y_aliases)
y_types_multi.extend(y_types)
y_hco_list_multi.extend(y_hco_list)
len_y_terms_multi = len(y_terms_multi)
ext_len = len(y_terms_multi) - len(y_terms)
for x_value, y_value_tuple in data:
try:
cur_y = y_values_multi[x_value]
cur_y.extend(y_value_tuple)
except KeyError:
y_values_multi[x_value] = [None]*ext_len
y_values_multi[x_value]\
.extend(y_value_tuple)
for _y_vals in y_values_multi.values():
if len(_y_vals) != len_y_terms_multi:
_y_vals.extend([None]*len(y_terms))
if y_terms_multi:
hco_x_axis = self.hcoptions['xAxis']
if len(hco_x_axis) - 1 < x_axis_num:
hco_x_axis\
.extend([HCOptions({})] *
(x_axis_num - (len(hco_x_axis)-1)))
hco_x_axis[x_axis_num]['categories'] = []
if x_mts:
if x_mapf:
data = ((x_mapf(x_value), y_vals) for
(x_value, y_vals) in
y_values_multi.iteritems())
sort_key = ((lambda(x, y): x_sortf(x)) if x_sortf
is not None else None)
data = sorted(data, key=sort_key)
else:
data = y_values_multi.iteritems()
sort_key = ((lambda(x, y): x_sortf(x)) if x_sortf
is not None else None)
data = sorted(data, key=sort_key)
if x_mapf:
data = [(x_mapf(x), y) for (x, y) in data]
for x_value, y_vals in data:
hco_x_axis[x_axis_num]['categories']\
.append(x_value)
for opts, y_value in izip(y_hco_list_multi, y_vals):
opts['data'].append(y_value)
self.hcoptions['series'].extend(y_hco_list_multi)
{'title': 'Cannot log in to Firefox Home App ',
'url': '/kb/Cannot+log+in+to+Firefox+Home+App'},
{'title': 'Replace your Sync information',
'url': '/kb/Replace+your+Sync+information'},
],
'tags': ['general'],
}),
('s5', {
'name': _('I have suggestions for how to improve Firefox Sync'),
'html': 'You can provide suggestions in our '
'<a href="http://assets2.getsatisfaction.com/mozilla_labs/products/mozilla_labs_weave_sync">'
'Firefox Sync feedback forum</a>.',
'deadend': True,
}),
])
}),
('other', {
'name': _("Thunderbird (Mozilla's email client) or other Mozilla product"),
'html': 'Support for Thunderbird and other Mozilla products can be found at'
' <a href="http://www.mozilla.org/support">Mozilla Support</a>.',
'categories': SortedDict([]),
'deadend': True,
}),
])
# Insert 'key' keys so we can go from product or category back to key:
for p_k, p_v in products.iteritems():
p_v['key'] = p_k
for c_k, c_v in p_v['categories'].iteritems():
c_v['key'] = c_k
class FieldSet(object):
"""
Data types fields definition.
This class allows to define set of fields to build complex pages.
"""
def __init__(self, force_order=None, **fields):
"""Init method.
@fields defines name/values for fields on this set, values
can be another FieldSet instance, alloweing to build complex
tree like structures, or a BaseType which will be the leaves
of the tree and final fields.
@force_order is a list of names to define the order that
fields should be presented on forms and other areas.
"""
# check that each field value is a BaseType of FieldSet instance
for value in fields.values():
if not isinstance(value, (BaseType, FieldSet)):
raise ValueError('%s is not a valid type' % value)
self.data = None # loaded data should be stored here
self.cache = {} # store converted values from data types
self.loaded = False # flag to mark that data was loaded
self.fields = SortedDict() # fields
for name in _ordering(force_order, fields): # store fields ordered
self.fields[name] = fields[name]
def inc_form(self, *names):
"""Include form method, must return a dict with context
values and cannot miss tpl key with template path that will
render the form."""
data = self._inc_form(*names)
if data and 'tpl' not in data:
raise ValueError('Missing form template')
return data
def _inc_form(self, *names):
"""Return form data needed to render form. Implement in subclass"""
raise NotImplementedError('Implement in subclass')
def load(self, values):
"""Load data into fields.
Relies on subclasses _load method to do the propper task. Won't load if
data was already loaded."""
if not self.loaded:
self._load(values)
self.loaded = True
def _load(self, values):
"""Load data into fields, implement in subclass"""
raise NotImplementedError('Implement in subclass')
def clone(self):
"""Clone FieldSet, sub FieldSets will be cloned too if any."""
fields = dict((k, v.clone() if isinstance(v, FieldSet) else v)
for k, v in self.fields.iteritems())
return self.__class__(force_order=self.fields.keys(), **fields)
def done_percent(self):
"""Return fields done percent."""
vals = [item.done_percent() for item in self.fields.values()]
return reduce(_reduce_pairs, vals) if vals else (len(self), 0)
def __len__(self):
"""Return loaded data lenght"""
return self.loaded and len(self.data) or 0
def __nonzero__(self):
"""Must returns FieldSet filled status, no fields or data means
not filled. Implement in subclass."""
if self.loaded:
data = any(bool(v) for v in self.data.itervalues()) \
if self.data else False
fields = any(bool(v) for v in self.fields.itervalues()
if isinstance(v, FieldSet)) \
if self.fields else False
return data or fields
else:
return False
def __hasitem__(self, name):
return name in self.fields
def __getitem__(self, name):
"""Dict like accessor to section fields. Sub FieldSets are
returned directly while BaseType values are proccesed by
it's type value method first."""
if name in self.fields:
if self.loaded:
item = self.fields[name]
if isinstance(item, BaseType):
if name not in self.cache:
self.cache[name] = item.value(self.data.get(name))
return self.cache[name]
elif isinstance(item, FieldSet):
return item
else:
return None
else:
raise KeyError('Missing value "%s"' % name)
def __getattr__(self, name):
try:
return self.__getitem__(name)
except KeyError, e:
raise AttributeError(str(e))
'name':
_lazy(u'I have suggestions for how to improve Firefox Sync'),
'html':
'You can provide suggestions in our '
'<a href="http://assets2.getsatisfaction.com/mozilla_labs/products/mozilla_labs_weave_sync">'
'Firefox Sync feedback forum</a>.',
'deadend':
True,
}),
])
}),
('other', {
'name':
_lazy(
u"Thunderbird (Mozilla's email client) or other Mozilla product"),
'html':
'Support for Thunderbird and other Mozilla products can be found at'
' <a href="http://www.mozilla.org/support">Mozilla Support</a>.',
'categories':
SortedDict([]),
'deadend':
True,
}),
])
# Insert 'key' keys so we can go from product or category back to key:
for p_k, p_v in products.iteritems():
p_v['key'] = p_k
for c_k, c_v in p_v['categories'].iteritems():
c_v['key'] = c_k
class Form(object):
def __init__(self, name, serializer_class, template_name=None, template_name_ro=None, auto_id="id_form_%s"):
self.name=name
self.auto_id=auto_id
self._fields= SortedDict()
fields=serializer_class().fields
for f in fields:
self.add_field(f, fields[f])
self.template_name= template_name or 'rest2backbone/form.html'
self.template_name_ro= template_name_ro or 'rest2backbone/form_ro.html'
self.force_ro=False
def add_field(self, name, serializer_field ):
self._fields[name]=Field(self,name, serializer_field)
@property
def id(self):
return self.auto_id%self.name.lower()
@property
def fields(self):
if self.force_ro:
fields=[copy.copy(f) for m,f in self._fields.iteritems()]
for form in fields:
form.force_ro=True
else:
fields=[f for m,f in self._fields.iteritems() if (not f.read_only and not f.fields)]
return fields
def get(self, name):
return self._fields.get(name)
def render_ro(self):
self.force_ro=True
try:
res=[]
res.append(u'<script type="text/template" id="r2b_template_%s_ro">'% self.name.lower())
res.append(render_to_string(self.template_name_ro, {'form':self}))
res.append(u'</script>')
return mark_safe('\n'.join(res))
finally:
self.force_ro=False
JS_TEMPLATE0="""if (!formsAPI.forms['%(form_id)s']) { formsAPI.forms['%(form_id)s']={} };"""
JS_TEMPLATE="""formsAPI.forms['%(form_id)s']['%(field_id)s']={cls:%(cls)s, options:%(opts)s};"""
def render(self):
res=[]
res.append(u'<script type="text/template" id="r2b_template_%s">'% self.name.lower())
t=render_to_string(self.template_name, {'form':self})
res.append(t)
res.append(u'</script>')
js=[]
zero_line=False
for f in self.fields:
cls,opts=f.render_js()
if cls:
if not zero_line:
js.append(self.JS_TEMPLATE0 % {'form_id':self.id})
zero_line=True
script=self.JS_TEMPLATE% {'cls':cls, 'opts':opts,
'field_id':f.id, 'form_id':self.id}
js.append(script)
if js:
res.append('<script>')
res.append('\n'.join(js))
res.append('</script>')
return mark_safe('\n'.join(res))
class FormContainer(StrAndUnicode):
__metaclass__ = FormContainerMetaclass
def __init__(self, **kwargs):
#
# Make self.forms from the metaclass-added self.form_classes
# and from instances determined by self.get_form_kwargs
#
# When SubscriberForm is inheriting, **kwargs are determined by Django and are
# {'files': None, 'prefix': 'subscriber', 'initial': {}, 'data': None}
#
self._errors = {}
self.forms = SortedDict()
container_prefix = kwargs.pop('prefix', '')
#
# In self.form_classes dictionary, there are members declared as
# form or formset classes in descendant class.
# When SubscriberForm derives from us, these will be
# {'server_info': FileServersInfoForm,
# 'subscriber': SubscriberFormFormSet,
# 'user': UserForm,
# 'broker_credentials': BrokerCredentialsFormFormSet}
#
# The resulting self.forms dictionary will contain them
# with instances determined by get_form_kwargs that adds 'instance'
# key to form kwargs before instantiation
#
# Instantiate all the forms in the container
for form_prefix, form_class in self.form_classes.items():
#
# For each pair
# form_prefix => form_class in form_classes dict
# a pair is constructed in self.forms dictionary
# form_prefix => form_class(**kwargs + {'instance': <added by get_form_kwargs>})
# The instantiated form is with prefix container_prefix "-" form_prefix
#
logger.debug("%s; form_prefix, form_class are %s, %s" % ('shree', form_prefix, form_class.__name__))
self.forms[form_prefix] = form_class(
prefix='-'.join(p for p in [container_prefix, form_prefix] if p),
**self.get_form_kwargs(form_prefix, **kwargs)
)
logger.debug("%s; forms[%s].prefix = %s" % ('shree', form_prefix, self.forms[form_prefix].prefix))
logger.debug('%s; self.forms = %s' % ('shree', self.forms))
pass
def __unicode__(self):
"Render all the forms in the container"
return mark_safe(u''.join([f.as_table() for f in self.forms.values()]))
def __iter__(self):
"Return each of the forms in the container"
for prefix in self.forms:
yield self[prefix]
def __getitem__(self, prefix):
"Return a specific form in the container"
try:
form = self.forms[prefix]
except KeyError:
raise KeyError('Prefix %r not found in Form container' % prefix)
return form
def is_valid(self):
return all(f.is_valid() for f in self.forms.values())
@property
def data(self):
"Return a compressed dictionary of all data from all subforms"
all_data = MultiValueDict()
for prefix, form in self.forms.items():
for key in form.data:
all_data.setlist(key, form.data.getlist(key))
return all_data
@property
def files(self):
"Return a compressed dictionary of all files from all subforms"
all_files = MultiValueDict()
for prefix, form in self.forms.items():
for key in form.files:
all_files.setlist(key, form.files.getlist(key))
return all_files
@property
def errors(self):
"Return a compressed dictionary of all errors form all subforms"
return dict((prefix, form.errors) for prefix, form in self.forms.items())
@property
def cleaned_data(self):
return {
prefix: form.cleaned_data
for prefix, form in self.forms.iteritems()
}
def save(self, *args, **kwargs):
"Save each of the subforms"
r = []
saved = SortedDict()
for p, f in self.forms.iteritems():
logger.debug('%s; saving form %s with cleaned data %s' % ('shree', f.__class__.__name__, f.cleaned_data))
s = f.save(*args, **kwargs)
logger.debug('%s; saved form %s' % ('shree', f.__class__.__name__))
r.append(s)
saved[p] = s
# Originally with comprehension
# r = [f.save(*args, **kwargs) for f in self.forms.values()]
# return r
return saved
def save_m2m(self):
"""Save any related objects -- e.g., m2m entries or inline formsets
This is needed if the original form collection was saved with commit=False
"""
for prefix, form in self.forms.items():
try:
for subform in form.saved_forms:
# Because the related instance wasn't saved at the time the
# form was created, the new PK value hasn't propegated to
# the inline object on the formset. We need to re-set the
# instance to update the _id attribute, which will allow the
# inline form instance to save.
setattr(subform.instance, form.fk.name, form.instance)
subform.instance.save()
except AttributeError:
pass
try:
form.save_m2m()
except AttributeError:
pass
class Collector(object):
def __init__(self, using):
self.using = using
# Initially, {model: set([instances])}, later values become lists.
self.data = {}
self.batches = {} # {model: {field: set([instances])}}
self.field_updates = {} # {model: {(field, value): set([instances])}}
self.dependencies = {} # {model: set([models])}
def add(self, objs, source=None, nullable=False, reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted. If the call is
the result of a cascade, 'source' should be the model that caused it,
and 'nullable' should be set to True if the relation can be null.
Returns a list of all objects that were not already collected.
"""
if not objs:
return []
new_objs = []
model = objs[0].__class__
instances = self.data.setdefault(model, set())
for obj in objs:
if obj not in instances:
new_objs.append(obj)
instances.update(new_objs)
# Nullable relationships can be ignored -- they are nulled out before
# deleting, and therefore do not affect the order in which objects have
# to be deleted.
if source is not None and not nullable:
if reverse_dependency:
source, model = model, source
self.dependencies.setdefault(source, set()).add(model)
return new_objs
def add_batch(self, model, field, objs):
"""
Schedules a batch delete. Every instance of 'model' that is related to
an instance of 'obj' through 'field' will be deleted.
"""
self.batches.setdefault(model, {}).setdefault(field, set()).update(objs)
def add_field_update(self, field, value, objs):
"""
Schedules a field update. 'objs' must be a homogenous iterable
collection of model instances (e.g. a QuerySet).
"""
if not objs:
return
model = objs[0].__class__
self.field_updates.setdefault(
model, {}).setdefault(
(field, value), set()).update(objs)
def collect(self, objs, source=None, nullable=False, collect_related=True,
source_attr=None, reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted as well as all
parent instances. 'objs' must be a homogenous iterable collection of
model instances (e.g. a QuerySet). If 'collect_related' is True,
related objects will be handled by their respective on_delete handler.
If the call is the result of a cascade, 'source' should be the model
that caused it and 'nullable' should be set to True, if the relation
can be null.
If 'reverse_dependency' is True, 'source' will be deleted before the
current model, rather than after. (Needed for cascading to parent
models, the one case in which the cascade follows the forwards
direction of an FK rather than the reverse direction.)
"""
if not connections[self.using].features.supports_deleting_related_objects:
collect_related = False
new_objs = self.add(objs, source, nullable,
reverse_dependency=reverse_dependency)
if not new_objs:
return
model = new_objs[0].__class__
# Recursively collect parent models, but not their related objects.
# These will be found by meta.get_all_related_objects()
for parent_model, ptr in model._meta.parents.iteritems():
if ptr:
parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
self.collect(parent_objs, source=model,
source_attr=ptr.rel.related_name,
collect_related=False,
reverse_dependency=True)
if collect_related:
for related in model._meta.get_all_related_objects(include_hidden=True):
field = related.field
if related.model._meta.auto_created:
self.add_batch(related.model, field, new_objs)
else:
sub_objs = self.related_objects(related, new_objs)
if not sub_objs:
continue
field.rel.on_delete(self, field, sub_objs, self.using)
# TODO This entire block is only needed as a special case to
# support cascade-deletes for GenericRelation. It should be
# removed/fixed when the ORM gains a proper abstraction for virtual
# or composite fields, and GFKs are reworked to fit into that.
for relation in model._meta.many_to_many:
if not relation.rel.through:
sub_objs = relation.bulk_related_objects(new_objs, self.using)
self.collect(sub_objs,
source=model,
source_attr=relation.rel.related_name,
nullable=True)
def related_objects(self, related, objs):
"""
Gets a QuerySet of objects related to ``objs`` via the relation ``related``.
"""
return related.model._base_manager.using(self.using).filter(
**{"%s__in" % related.field.name: objs}
)
def instances_with_model(self):
for model, instances in self.data.iteritems():
for obj in instances:
yield model, obj
def sort(self):
sorted_models = []
models = self.data.keys()
while len(sorted_models) < len(models):
found = False
for model in models:
if model in sorted_models:
continue
dependencies = self.dependencies.get(model)
if not (dependencies and dependencies.difference(sorted_models)):
sorted_models.append(model)
found = True
if not found:
return
self.data = SortedDict([(model, self.data[model])
for model in sorted_models])
@force_managed
def delete(self):
# sort instance collections
for model, instances in self.data.items():
self.data[model] = sorted(instances, key=attrgetter("pk"))
# if possible, bring the models in an order suitable for databases that
# don't support transactions or cannot defer contraint checks until the
# end of a transaction.
self.sort()
# send pre_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.pre_delete.send(
sender=model, instance=obj, using=self.using
)
# update fields
for model, instances_for_fieldvalues in self.field_updates.iteritems():
query = sql.UpdateQuery(model)
for (field, value), instances in instances_for_fieldvalues.iteritems():
query.update_batch([obj.pk for obj in instances],
{field.name: value}, self.using)
# reverse instance collections
for instances in self.data.itervalues():
instances.reverse()
# delete batches
for model, batches in self.batches.iteritems():
query = sql.DeleteQuery(model)
for field, instances in batches.iteritems():
query.delete_batch([obj.pk for obj in instances], self.using, field)
# delete instances
for model, instances in self.data.iteritems():
query = sql.DeleteQuery(model)
pk_list = [obj.pk for obj in instances]
query.delete_batch(pk_list, self.using)
# send post_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.post_delete.send(
sender=model, instance=obj, using=self.using
)
# update collected instances
for model, instances_for_fieldvalues in self.field_updates.iteritems():
for (field, value), instances in instances_for_fieldvalues.iteritems():
for obj in instances:
setattr(obj, field.attname, value)
for model, instances in self.data.iteritems():
for instance in instances:
setattr(instance, model._meta.pk.attname, None)
class XmlObjectForm(BaseForm):
"""Django Form based on an :class:`~eulcore.xmlmap.XmlObject` model,
analogous to Django's ModelForm.
Note that not all :mod:`eulcore.xmlmap.fields` are currently supported; all
released field types are supported in their single-node variety, but no list
field types are currently supported. Attempting to define an XmlObjectForm
without excluding unsupported fields will result in an Exception.
Unlike Django's ModelForm, which provides a save() method, XmlObjectForm
provides analogous functionality via :meth:`update_instance`. Since an
XmlObject by itself does not have a save method, and can only be saved in
particular contexts (e.g., :mod:`eulcore.existdb` or :mod:`eulcore.fedora`),
there is no meaningful way for an XmlObjectForm to save an associated model
instance to the appropriate datastore.
If you wish to customize the html display for an XmlObjectForm, rather than
using the built-in form display functions, be aware that if your XmlObject
has any fields of type :class:`~eulcore.xmlmap.fields.NodeField`, you should
make sure to display the subforms for those fields.
NOTE: If your XmlObject includes NodeField elements and you do not want
empty elements in your XML output when empty values are entered into the form,
you may wish to extend :meth:`eulcore.xmlmap.XmlObject.is_empty` to correctly
identify when your NodeField elements should be considered empty (if the
default definition is not accurate or appropriate). Empty elements will not
be added to the :class:``eulcore.xmlmap.XmlObject` instance returned by
:meth:`update_instance`.
"""
# django has a basemodelform with all the logic
# and then a modelform with the metaclass declaration; do we need that?
__metaclass__ = XmlObjectFormType
_html_section = None # formatting for outputting object with subform
subforms = {}
"""Sorted Dictionary of :class:`XmlObjectForm` instances for fields of type
:class:`~eulcore.xmlmap.fields.NodeField` belonging to this Form's
:class:`~eulcore.xmlmap.XmlObject` model, keyed on field name. Ordered by
field creation order or by specified fields."""
form_label = None
'''Label for this form or subform (set automatically for subforms &
formsets, using the same logic that is used for field labels.'''
def __init__(self, data=None, instance=None, prefix=None, initial={}, **kwargs):
opts = self._meta
# make a copy of any initial data for local use, since it may get updated with instance data
local_initial = initial.copy()
if instance is None:
if opts.model is None:
raise ValueError('XmlObjectForm has no XmlObject model class specified')
# if we didn't get an instance, instantiate a new one
# NOTE: if this is a subform, the data won't go anywhere useful
# currently requires that instantiate_on_get param be set to True for NodeFields
self.instance = opts.model()
# track adding new instance instead of updating existing?
else:
self.instance = instance
# generate dictionary of initial data based on current instance
# allow initial data from instance to co-exist with other initial data
local_initial.update(xmlobject_to_dict(self.instance)) #, prefix=prefix)) # fields, exclude?
# FIXME: is this backwards? should initial data override data from instance?
# initialize subforms for all nodefields that belong to the xmlobject model
self._init_subforms(data, prefix)
self._init_formsets(data, prefix)
super_init = super(XmlObjectForm, self).__init__
super_init(data=data, prefix=prefix, initial=local_initial, **kwargs)
# other kwargs accepted by XmlObjectForm.__init__:
# files, auto_id, object_data,
# error_class, label_suffix, empty_permitted
def _init_subforms(self, data=None, prefix=None):
# initialize each subform class with the appropriate model instance and data
self.subforms = SortedDict() # create as sorted dictionary to preserve order
for name, subform in self.__class__.subforms.iteritems():
# instantiate the new form with the current field as instance, if available
if self.instance is not None:
# get the relevant instance for the current NodeField variable
# NOTE: calling create_foo will create the nodefield for element foo
# creating here so subfields will be set correctly
# if the resulting field is empty, it will be removed by update_instance
getattr(self.instance, 'create_' + name)()
subinstance = getattr(self.instance, name, None)
else:
subinstance = None
if prefix:
subprefix = '%s-%s' % (prefix, name)
else:
subprefix = name
# instantiate the subform class with field data and model instance
# - setting prefix based on field name, to distinguish similarly named fields
newform = subform(data=data, instance=subinstance, prefix=subprefix)
# depending on how the subform was declared, it may not have a label yet
if newform.form_label is None:
if name in self.subform_labels:
newform.form_label = self.subform_labels[name]
self.subforms[name] = newform
def _init_formsets(self, data=None, prefix=None):
self.formsets = {}
for name, formset in self.__class__.formsets.iteritems():
if self.instance is not None:
subinstances = getattr(self.instance, name, None)
else:
subinstances = None
if prefix is not None:
subprefix = '%s-%s' % (prefix, name)
else:
subprefix = name
self.formsets[name] = formset(data=data, instances=subinstances, prefix=subprefix)
def update_instance(self):
"""Save bound form data into the XmlObject model instance and return the
updated instance."""
# NOTE: django model form has a save method - not applicable here,
# since an XmlObject by itself is not expected to have a save method
# (only likely to be saved in context of a fedora or exist object)
if hasattr(self, 'cleaned_data'): # possible to have an empty object/no data
opts = self._meta
for name in self.instance._fields.iterkeys():
if opts.fields and name not in opts.parsed_fields.fields:
continue
if opts.exclude and name in opts.parsed_exclude.fields:
continue
if name in self.cleaned_data:
# special case: we don't want empty attributes and elements
# for fields which returned no data from the form
# converting '' to None and letting XmlObject handle
if self.cleaned_data[name] == '':
self.cleaned_data[name] = None
setattr(self.instance, name, self.cleaned_data[name])
# update sub-model portions via any subforms
for name, subform in self.subforms.iteritems():
self._update_subinstance(name, subform)
for formset in self.formsets.itervalues():
formset.update_instance()
return self.instance
def _update_subinstance(self, name, subform):
"""Save bound data for a single subform into the XmlObject model
instance."""
old_subinstance = getattr(self.instance, name)
new_subinstance = subform.update_instance()
# if our instance previously had no node for the subform AND the
# updated one has data, then attach the new node.
if old_subinstance is None and not new_subinstance.is_empty():
setattr(self.instance, name, new_subinstance)
# on the other hand, if the instance previously had a node for the
# subform AND the updated one is empty, then remove the node.
if old_subinstance is not None and new_subinstance.is_empty():
delattr(self.instance, name)
def is_valid(self):
"""Returns True if this form and all subforms (if any) are valid.
If all standard form-validation tests pass, uses :class:`~eulcore.xmlmap.XmlObject`
validation methods to check for schema-validity (if a schema is associated)
and reporting errors. Additonal notes:
* schema validation requires that the :class:`~eulcore.xmlmap.XmlObject`
be initialized with the cleaned form data, so if normal validation
checks pass, the associated :class:`~eulcore.xmlmap.XmlObject` instance
will be updated with data via :meth:`update_instance`
* schema validation errors SHOULD NOT happen in a production system
:rtype: boolean
"""
valid = super(XmlObjectForm, self).is_valid() and \
all(s.is_valid() for s in self.subforms.itervalues()) and \
all(s.is_valid() for s in self.formsets.itervalues())
# schema validation can only be done after regular validation passes,
# because xmlobject must be updated with cleaned_data
if valid and self.instance is not None:
# update instance required to check schema-validity
instance = self.update_instance()
if instance.is_valid():
return True
else:
# if not schema-valid, add validation errors to error dictionary
# NOTE: not overriding _get_errors because that is used by the built-in validation
# append to any existing non-field errors
if NON_FIELD_ERRORS not in self._errors:
self._errors[NON_FIELD_ERRORS] = self.error_class()
self._errors[NON_FIELD_ERRORS].append("There was an unexpected schema validation error. " +
"This should not happen! Please report the following errors:")
for err in instance.validation_errors():
self._errors[NON_FIELD_ERRORS].append('VALIDATION ERROR: %s' % err.message)
return False
return valid
# NOTE: errors only returned for the *current* form, not for all subforms
# - appears to be used only for form output, so this should be sensible
def _html_output(self, normal_row, error_row, row_ender, help_text_html, errors_on_separate_row):
"""Extend BaseForm's helper function for outputting HTML. Used by as_table(), as_ul(), as_p().
Combines the HTML version of the main form's fields with the HTML content
for any subforms.
"""
parts = []
parts.append(super(XmlObjectForm, self)._html_output(normal_row, error_row, row_ender,
help_text_html, errors_on_separate_row))
def _subform_output(subform):
return subform._html_output(normal_row, error_row, row_ender,
help_text_html, errors_on_separate_row)
for name, subform in self.subforms.iteritems():
parts.append(self._html_subform_output(subform, name, _subform_output))
for name, formset in self.formsets.iteritems():
parts.append(unicode(formset.management_form))
# use form label if one was set
if hasattr(formset, 'form_label'):
name = formset.form_label
# collect the html output for all the forms in the formset
subform_parts = list()
for subform in formset.forms:
subform_parts.append(self._html_subform_output(subform,
gen_html=_subform_output, suppress_section=True))
# then wrap all forms in the section container, so formset label appears once
parts.append(self._html_subform_output(name=name, content=u'\n'.join(subform_parts)))
return mark_safe(u'\n'.join(parts))
def _html_subform_output(self, subform=None, name=None, gen_html=None, content=None,
suppress_section=False):
# pass the configured html section to subform in case of any sub-subforms
if subform is not None:
subform._html_section = self._html_section
if gen_html is not None:
content = gen_html(subform)
# if html section is configured, add section label and wrapper for
if self._html_section is not None and not suppress_section:
return self._html_section % \
{'label': fieldname_to_label(name), 'content': content}
else:
return content
# intercept the three standard html output formats to set an appropriate section format
def as_table(self):
"""Behaves exactly the same as Django Form's as_table() method,
except that it also includes the fields for any associated subforms
in table format.
Subforms, if any, will be grouped in a <tbody> labeled with a heading
based on the label of the field.
"""
self._html_section = u'<tbody><tr><th colspan="2" class="section">%(label)s</th></tr><tr><td colspan="2"><table class="subform">\n%(content)s</table></td></tr></tbody>'
#self._html_section = u'<tbody><tr><th class="section" colspan="2">%(label)s</th></tr>\n%(content)s</tbody>'
return super(XmlObjectForm, self).as_table()
def as_p(self):
"""Behaves exactly the same as Django Form's as_p() method,
except that it also includes the fields for any associated subforms
in paragraph format.
Subforms, if any, will be grouped in a <div> of class 'subform',
with a heading based on the label of the field.
"""
self._html_section = u'<div class="subform"><p class="label">%(label)s</p>%(content)s</div>'
return super(XmlObjectForm, self).as_p()
def as_ul(self):
"""Behaves exactly the same as Django Form's as_ul() method,
except that it also includes the fields for any associated subforms
in list format.
Subforms, if any, will be grouped in a <ul> of class 'subform',
with a heading based on the label of the field.
"""
self._html_section = u'<li class="subform"><p class="label">%(label)s</p><ul>%(content)s</ul></li>'
return super(XmlObjectForm, self).as_ul()
class Collector(object):
def __init__(self, using):
self.using = using
# Initially, {model: set([instances])}, later values become lists.
self.data = {}
self.batches = {} # {model: {field: set([instances])}}
self.field_updates = {} # {model: {(field, value): set([instances])}}
self.dependencies = {} # {model: set([models])}
def add(self, objs, source=None, nullable=False, reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted. If the call is
the result of a cascade, 'source' should be the model that caused it
and 'nullable' should be set to True, if the relation can be null.
Returns a list of all objects that were not already collected.
"""
if not objs:
return []
new_objs = []
model = objs[0].__class__
instances = self.data.setdefault(model, set())
for obj in objs:
if obj not in instances:
new_objs.append(obj)
instances.update(new_objs)
# Nullable relationships can be ignored -- they are nulled out before
# deleting, and therefore do not affect the order in which objects have
# to be deleted.
if new_objs and source is not None and not nullable:
if reverse_dependency:
source, model = model, source
self.dependencies.setdefault(source, set()).add(model)
return new_objs
def add_batch(self, model, field, objs):
"""
Schedules a batch delete. Every instance of 'model' that is related to
an instance of 'obj' through 'field' will be deleted.
"""
self.batches.setdefault(model, {}).setdefault(field,
set()).update(objs)
def add_field_update(self, field, value, objs):
"""
Schedules a field update. 'objs' must be a homogenous iterable
collection of model instances (e.g. a QuerySet).
"""
if not objs:
return
model = objs[0].__class__
self.field_updates.setdefault(model, {}).setdefault((field, value),
set()).update(objs)
def collect(self,
objs,
source=None,
nullable=False,
collect_related=True,
source_attr=None,
reverse_dependency=False):
"""
Adds 'objs' to the collection of objects to be deleted as well as all
parent instances. 'objs' must be a homogenous iterable collection of
model instances (e.g. a QuerySet). If 'collect_related' is True,
related objects will be handled by their respective on_delete handler.
If the call is the result of a cascade, 'source' should be the model
that caused it and 'nullable' should be set to True, if the relation
can be null.
If 'reverse_dependency' is True, 'source' will be deleted before the
current model, rather than after. (Needed for cascading to parent
models, the one case in which the cascade follows the forwards
direction of an FK rather than the reverse direction.)
"""
new_objs = self.add(objs,
source,
nullable,
reverse_dependency=reverse_dependency)
if not new_objs:
return
model = new_objs[0].__class__
# Recursively collect parent models, but not their related objects.
# These will be found by meta.get_all_related_objects()
for parent_model, ptr in model._meta.parents.iteritems():
if ptr:
parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
self.collect(parent_objs,
source=model,
source_attr=ptr.rel.related_name,
collect_related=False,
reverse_dependency=True)
if collect_related:
for related in model._meta.get_all_related_objects(
include_hidden=True):
field = related.field
if related.model._meta.auto_created:
self.add_batch(related.model, field, new_objs)
else:
sub_objs = self.related_objects(related, new_objs)
if not sub_objs:
continue
field.rel.on_delete(self, field, sub_objs, self.using)
# TODO This entire block is only needed as a special case to
# support cascade-deletes for GenericRelation. It should be
# removed/fixed when the ORM gains a proper abstraction for virtual
# or composite fields, and GFKs are reworked to fit into that.
for relation in model._meta.many_to_many:
if not relation.rel.through:
sub_objs = relation.bulk_related_objects(
new_objs, self.using)
self.collect(sub_objs,
source=model,
source_attr=relation.rel.related_name,
nullable=True)
def related_objects(self, related, objs):
"""
Gets a QuerySet of objects related to ``objs`` via the relation ``related``.
"""
return related.model._base_manager.using(
self.using).filter(**{"%s__in" % related.field.name: objs})
def instances_with_model(self):
for model, instances in self.data.iteritems():
for obj in instances:
yield model, obj
def sort(self):
sorted_models = []
models = self.data.keys()
while len(sorted_models) < len(models):
found = False
for model in models:
if model in sorted_models:
continue
dependencies = self.dependencies.get(model)
if not (dependencies
and dependencies.difference(sorted_models)):
sorted_models.append(model)
found = True
if not found:
return
self.data = SortedDict([(model, self.data[model])
for model in sorted_models])
@force_managed
def delete(self):
# sort instance collections
for model, instances in self.data.items():
self.data[model] = sorted(instances, key=attrgetter("pk"))
# if possible, bring the models in an order suitable for databases that
# don't support transactions or cannot defer contraint checks until the
# end of a transaction.
self.sort()
# send pre_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.pre_delete.send(sender=model,
instance=obj,
using=self.using)
# update fields
for model, instances_for_fieldvalues in self.field_updates.iteritems():
query = sql.UpdateQuery(model)
for (field,
value), instances in instances_for_fieldvalues.iteritems():
query.update_batch([obj.pk for obj in instances],
{field.name: value}, self.using)
# reverse instance collections
for instances in self.data.itervalues():
instances.reverse()
# delete batches
for model, batches in self.batches.iteritems():
query = sql.DeleteQuery(model)
for field, instances in batches.iteritems():
query.delete_batch([obj.pk for obj in instances], self.using,
field)
# delete instances
for model, instances in self.data.iteritems():
query = sql.DeleteQuery(model)
pk_list = [obj.pk for obj in instances]
query.delete_batch(pk_list, self.using)
# send post_delete signals
for model, obj in self.instances_with_model():
if not model._meta.auto_created:
signals.post_delete.send(sender=model,
instance=obj,
using=self.using)
# update collected instances
for model, instances_for_fieldvalues in self.field_updates.iteritems():
for (field,
value), instances in instances_for_fieldvalues.iteritems():
for obj in instances:
setattr(obj, field.attname, value)
for model, instances in self.data.iteritems():
for instance in instances:
setattr(instance, model._meta.pk.attname, None)
def generate_plot(self):
# reset the series
self.hcoptions['series'] = []
dss = self.datasource.series
# find all x's from different datasources that need to be plotted on
# same xAxis and also find their corresponding y's
cht_typ_grp = lambda y_term: ('scatter' if self.series_options[y_term][
'type'] in ['scatter', 'pie'] else 'line')
for x_axis_num, vqs_groups in self.x_axis_vqs_groups.items():
y_hco_list = []
try:
x_sortf, x_mapf, x_mts = self.x_sortf_mapf_mts[x_axis_num]
except IndexError:
x_sortf, x_mapf, x_mts = (None, None, False)
ptype_x_y_terms = defaultdict(list)
for vqs_group in vqs_groups.values():
x_term, y_terms_all = vqs_group.items()[0]
y_terms_by_type = defaultdict(list)
for y_term in y_terms_all:
y_terms_by_type[cht_typ_grp(y_term)].append(y_term)
for y_type, y_term_list in y_terms_by_type.items():
ptype_x_y_terms[y_type].append((x_term, y_term_list))
# ptype = plot type i.e. 'line', 'scatter', 'area', etc.
for ptype, x_y_terms_tuples in ptype_x_y_terms.items():
y_fields_multi = []
y_aliases_multi = []
y_types_multi = []
y_hco_list_multi = []
y_values_multi = SortedDict()
y_terms_multi = []
for x_term, y_terms in x_y_terms_tuples:
# x related
x_vqs = dss[x_term]['_data']
x_field = dss[x_term]['field']
# y related
y_fields = [dss[y_term]['field'] for y_term in y_terms]
y_aliases = [
dss[y_term]['field_alias'] for y_term in y_terms
]
y_types = [
self.series_options[y_term].get('type', 'line')
for y_term in y_terms
]
y_hco_list = [
HCOptions(copy.deepcopy(self.series_options[y_term]))
for y_term in y_terms
]
for opts, alias, typ in zip(y_hco_list, y_aliases,
y_types):
opts.pop('_x_axis_term')
opts['name'] = alias
opts['type'] = typ
opts['data'] = []
if ptype == 'scatter' or (ptype == 'line'
and len(x_y_terms_tuples) == 1):
if x_mts:
if x_mapf:
data = ((x_mapf(value_dict[x_field]), [
value_dict[y_field] for y_field in y_fields
]) for value_dict in x_vqs)
sort_key = ((lambda (x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(data, key=sort_key)
else:
sort_key = ((lambda (x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(((value_dict[x_field], [
value_dict[y_field] for y_field in y_fields
]) for value_dict in x_vqs),
key=sort_key)
if x_mapf:
data = [(x_mapf(x), y) for (x, y) in data]
if ptype == 'scatter':
#scatter plot and pie chart
for x_value, y_value_tuple in data:
for opts, y_value in izip(
y_hco_list, y_value_tuple):
opts['data'].append((x_value, y_value))
self.hcoptions['series'].extend(y_hco_list)
if ptype == 'line' and len(x_y_terms_tuples) == 1:
# all other chart types - line, area, etc.
hco_x_axis = self.hcoptions['xAxis']
if len(hco_x_axis) - 1 < x_axis_num:
hco_x_axis.extend([HCOptions({})] *
(x_axis_num -
(len(hco_x_axis) - 1)))
hco_x_axis[x_axis_num]['categories'] = []
for x_value, y_value_tuple in data:
hco_x_axis[x_axis_num]['categories']\
.append(x_value)
for opts, y_value in izip(
y_hco_list, y_value_tuple):
opts['data'].append(y_value)
self.hcoptions['series'].extend(y_hco_list)
else:
data = ((value_dict[x_field],
[value_dict[y_field] for y_field in y_fields])
for value_dict in x_vqs)
y_terms_multi.extend(y_terms)
y_fields_multi.extend(y_fields)
y_aliases_multi.extend(y_aliases)
y_types_multi.extend(y_types)
y_hco_list_multi.extend(y_hco_list)
len_y_terms_multi = len(y_terms_multi)
ext_len = len(y_terms_multi) - len(y_terms)
for x_value, y_value_tuple in data:
try:
cur_y = y_values_multi[x_value]
cur_y.extend(y_value_tuple)
except KeyError:
y_values_multi[x_value] = [None] * ext_len
y_values_multi[x_value]\
.extend(y_value_tuple)
for _y_vals in y_values_multi.values():
if len(_y_vals) != len_y_terms_multi:
_y_vals.extend([None] * len(y_terms))
if y_terms_multi:
hco_x_axis = self.hcoptions['xAxis']
if len(hco_x_axis) - 1 < x_axis_num:
hco_x_axis\
.extend([HCOptions({})]*
(x_axis_num - (len(hco_x_axis)-1)))
hco_x_axis[x_axis_num]['categories'] = []
if x_mts:
if x_mapf:
data = ((x_mapf(x_value), y_vals)
for (x_value,
y_vals) in y_values_multi.iteritems())
sort_key = ((lambda (x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(data, key=sort_key)
else:
data = y_values_multi.iteritems()
sort_key = ((lambda (x, y): x_sortf(x))
if x_sortf is not None else None)
data = sorted(data, key=sort_key)
if x_mapf:
data = [(x_mapf(x), y) for (x, y) in data]
for x_value, y_vals in data:
hco_x_axis[x_axis_num]['categories']\
.append(x_value)
for opts, y_value in izip(y_hco_list_multi, y_vals):
opts['data'].append(y_value)
self.hcoptions['series'].extend(y_hco_list_multi)
