def get_files(self): files = MultiValueDict({}) for step, step_data in self.request.session[self.prefix][self.step_data_session_key].items(): if step_data.has_key('files'): for file in step_data.getlist('files'): files.appendlist(step+'-file', self.file_storage.open(file.get('path'))) return files
def parse_file_upload(header_dict, post_data): "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" import email, email.Message from cgi import parse_header raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) raw_message += '\r\n\r\n' + post_data msg = email.message_from_string(raw_message) POST = MultiValueDict() FILES = MultiValueDict() for submessage in msg.get_payload(): if submessage and isinstance(submessage, email.Message.Message): name_dict = parse_header(submessage['Content-Disposition'])[1] # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads # or {'name': 'blah'} for POST fields # We assume all uploaded files have a 'filename' set. if name_dict.has_key('filename'): assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" if not name_dict['filename'].strip(): continue # IE submits the full path, so trim everything but the basename. # (We can't use os.path.basename because it expects Linux paths.) filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] FILES.appendlist(name_dict['name'], { 'filename': filename, 'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None), 'content': submessage.get_payload(), }) else: POST.appendlist(name_dict['name'], submessage.get_payload()) return POST, FILES
def test_encode_multipart_data_multiple_params(self): # Sequences of parameters and files can be passed to # encode_multipart_data() so that multiple parameters/files with the # same name can be provided. params_in = [ ("one", "ABC"), ("one", "XYZ"), ("two", "DEF"), ("two", "UVW"), ] files_in = [ ("f-one", BytesIO(urandom(32))), ("f-two", BytesIO(urandom(32))), ] body, headers = encode_multipart_data(params_in, files_in) self.assertEqual("%s" % len(body), headers["Content-Length"]) self.assertThat( headers["Content-Type"], StartsWith("multipart/form-data; boundary=")) # Round-trip through Django's multipart code. params_out, files_out = ( parse_headers_and_body_with_django(headers, body)) params_out_expected = MultiValueDict() for name, value in params_in: params_out_expected.appendlist(name, value) self.assertEqual( params_out_expected, params_out, ahem_django_ahem) self.assertSetEqual({"f-one", "f-two"}, set(files_out)) files_expected = {name: buf.getvalue() for name, buf in files_in} files_observed = {name: buf.read() for name, buf in files_out.items()} self.assertEqual( files_expected, files_observed, ahem_django_ahem)
def parse_file_upload(header_dict, post_data): """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" import email, email.Message from cgi import parse_header raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) raw_message += '\r\n\r\n' + post_data msg = email.message_from_string(raw_message) POST = QueryDict('', mutable=True) FILES = MultiValueDict() for submessage in msg.get_payload(): if submessage and isinstance(submessage, email.Message.Message): name_dict = parse_header(submessage['Content-Disposition'])[1] # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads # or {'name': 'blah'} for POST fields # We assume all uploaded files have a 'filename' set. if 'filename' in name_dict: assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" if not name_dict['filename'].strip(): continue # IE submits the full path, so trim everything but the basename. # (We can't use os.path.basename because that uses the server's # directory separator, which may not be the same as the # client's one.) filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] FILES.appendlist(name_dict['name'], FileDict({ 'filename': filename, 'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None, 'content': submessage.get_payload(), })) else: POST.appendlist(name_dict['name'], submessage.get_payload()) return POST, FILES
def status_update(self): self.debug('{0} running'.format(SCHEDULE_DESC)) now = datetime.datetime.now() notifications = TagNotification.objects.filter(sent=False) notifications = notifications.select_related().order_by('tag', 'entry') self.info('found {0} notifications'.format(notifications.count())) users = {} for notification in notifications: email = notification.user.email if email not in users: users[email] = [] users[email].append(notification) for email, notifications in users.iteritems(): tags = MultiValueDict() for notification in notifications: tags.appendlist(notification.tag, notification) context = {'tags': tags} body = render_to_string('tree/digest.txt', context) try: send_mail(subject='Survey Response Report', message=body, recipient_list=[email], from_email='*****@*****.**', fail_silently=False) sent = True except smtplib.SMTPException, e: self.exception(e) sent = False if sent: for notification in notifications: notification.sent = True notification.date_sent = datetime.datetime.now() notification.save() self.info('Sent report to %s' % email)
def resolve_params(self, text): " Parse the parameters into a dict. " params = MultiValueDict() for line in text.split('\n'): pair = line.split(':', 1) if len(pair) == 2: params.appendlist(pair[0].strip(), pair[1].strip()) return params
def _do_export(survey, subject_type, pk): facts = models.Fact.objects\ .filter(survey=survey, content_type=subject_type, object_id=pk)\ .select_related('desired_fact') export_data = MultiValueDict() for fact in facts: export_data.appendlist(fact.desired_fact.code, fact.data) return export_data
def jquery_to_dict(values): result = MultiValueDict() if not isinstance(values, (list, tuple, set)): return result for value in values: if not isinstance(value, dict): continue if 'name' in value and 'value' in value: result.appendlist(value['name'], value['value']) return result
def _build_sub_query(self, search_node): terms = MultiValueDict() for child in search_node.children: if isinstance(child, SearchNode): terms.update(self._build_sub_query(child)) else: terms.appendlist(child[0], child[1]) return terms
class MakeMonthArrayNode (Node): def __init__ (self): self.year = None self.month = None self.data = MultiValueDict() self._weekday = None # Weekday (Monday==0) for day 1 in this moth. self._n_days = None # Number of days in month. def _n_added_days(self): """Number of added days at the end of the month to have a "full week".""" return ( 7 - (self._weekday + self._n_days)%7 ) % 7 def __list2array__ (self, l, ncols=7): """Splits list l in lists of len()=cols. l must be "squareable". """ return [ l[row*ncols:(row+1)*ncols] for row in range(0, len(l)/ncols) ] def squared (self): """Inserts keys and values in an array of days (month's calendar). Returns a list of weeks. One week, one list of days. One day, one list with [ <day-number> , [<data for this day>] ] """ # Insert [None] in remaining days. IN PLACE. for i in range(1, self._n_days+1): if not self.data.has_key(i): self.data[i] = None # zerofill out-of-month days before = self._weekday * [[0,[None]]] after = self._n_added_days() * [[0,[None]]] # cat three lists and split per weeks return self.__list2array__ (before + self.data.lists() + after) def render(self, context): try: self.year = context['month'].year self.month = context['month'].month self._weekday, self._n_days = monthrange(self.year, self.month) for o in context['object_list']: self.data.appendlist (o.startdate.day, o) context['month_array'] = self.squared() except: pass return ''
def existing_facts(survey, subject, prefix=None): """return all facts for a given survey and subject as a MultiValueDict, so that it can be bound to a form. """ content_type = ContentType.objects.get_for_model(subject) facts = Fact.objects.filter(survey=survey, content_type=content_type, object_id=subject.id)\ .select_related('desired_fact') fmt = (prefix and "%s-%s" % (prefix, "%s")) or "%s" existing_facts = MultiValueDict() for fact in facts: code = fmt % fact.desired_fact.code existing_facts.appendlist(code, fact.data) return existing_facts
def replace_esi_tags(request, response): process_errors = getattr(settings, 'ESI_PROCESS_ERRORS', False) fragment_headers = MultiValueDict() fragment_cookies = [] request_data = { 'cookies': request.COOKIES, 'HTTP_REFERER': request.build_absolute_uri(), 'HTTP_X_ESI_FRAGMENT': True, } replacement_offset = 0 for match in esi_tag_re.finditer(response.content): url = build_full_fragment_url(request, match.group('url')) if response.status_code == 200 or process_errors: client = http_client.Client(**request_data) fragment = client.get(url) else: fragment = HttpResponse() if fragment.status_code != 200: extra = {'data': { 'fragment': fragment.__dict__, 'request': request.__dict__, }} log.error('ESI fragment %s returned status code %s' % (url, fragment.status_code), extra=extra) # Remove the error content so it isn't added to the page. fragment.content = '' start = match.start() + replacement_offset end = match.end() + replacement_offset response.content = '%s%s%s' % (response.content[:start], fragment.content, response.content[end:]) replacement_offset += len(fragment.content) - len(match.group(0)) for header in HEADERS_TO_MERGE: if header in fragment: fragment_headers.appendlist(header, fragment[header]) if fragment.cookies: fragment_cookies.append(fragment.cookies) merge_fragment_headers(response, fragment_headers) merge_fragment_cookies(response, fragment_cookies)
def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): super(AjaxUploadHandlerMixin, self).handle_raw_input(input_data, META, content_length, boundary, encoding) if META.get('HTTP_X_REQUESTED_WITH') != 'XMLHttpRequest': return # request is not an ajax request if not getattr(self, 'activated', True): return # file is too large for MemoryFileUploadHandler # filename is set as a header as well as a GET param file_name = META.get('HTTP_X_FILE_NAME', META.get('X_FILE_NAME')) if file_name is None: raise MultiPartParserError('Invalid File-Name: %r' % file_name) # clean the filename, taken from the default MultiPartParser file_name = force_unicode(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unquote(file_name)) if not file_name: raise MultiPartParserError('Invalid File-Name: %r' % file_name) # the javascript does not try to set the content_type # so we guess it from the filename. content_type, enc = mimetypes.guess_type(file_name) chunk_start = 0 try: self.new_file(self.field_name, file_name, content_type, content_length, encoding) except StopFutureHandlers: pass # expected from MemoryFileUploadHandler # feed the file to the real upload handler. for chunk in ChunkIter(input_data, self.chunk_size): chunk_length = len(chunk) self.receive_data_chunk(chunk, chunk_start) chunk_start += chunk_length file_obj = self.file_complete(chunk_start) # create the POST and FILES datastructures post = QueryDict(MultiValueDict(), encoding=encoding) files = MultiValueDict() if file_obj: files.appendlist(self.field_name, file_obj) return post, files
def test_encode_multipart_data_multiple_params(self): # Sequences of parameters and files passed to # encode_multipart_data() permit use of the same name for # multiple parameters and/or files. See `make_payloads` to # understand how it processes different types of parameter # values. params_in = [ ("one", "ABC"), ("one", "XYZ"), ("two", ["DEF", "UVW"]), ] files = [ BytesIO(b"f1"), open(self.make_file(contents=b"f2"), "rb"), open(self.make_file(contents=b"f3"), "rb"), ] for fd in files: self.addCleanup(fd.close) files_in = [ ("f-one", files[0]), ("f-two", files[1]), ("f-three", lambda: files[2]), ] body, headers = encode_multipart_data(params_in, files_in) self.assertEqual("%s" % len(body), headers["Content-Length"]) self.assertThat( headers["Content-Type"], StartsWith("multipart/form-data; boundary=")) # Round-trip through Django's multipart code. params_out, files_out = ( parse_headers_and_body_with_django(headers, body)) params_out_expected = MultiValueDict() params_out_expected.appendlist("one", "ABC") params_out_expected.appendlist("one", "XYZ") params_out_expected.appendlist("two", "DEF") params_out_expected.appendlist("two", "UVW") self.assertEqual( params_out_expected, params_out, ahem_django_ahem) files_expected = {"f-one": b"f1", "f-two": b"f2", "f-three": b"f3"} files_observed = {name: buf.read() for name, buf in files_out.items()} self.assertEqual( files_expected, files_observed, ahem_django_ahem)
def test_save_valid_multiple_choice(self): self.desired_fact.data_type = 'M' self.desired_fact.save() FactOption.objects.create(desired_fact=self.desired_fact, code='01', description='1') FactOption.objects.create(desired_fact=self.desired_fact, code='02', description='2') cls = forms.make_survey_form_subclass(self.survey, self.content_type) post_data = MultiValueDict() post_data.appendlist('code1', '01') post_data.appendlist('code1', '02') cls(self.survey, self.subject, self.user, data=post_data).save_valid() self.assertTrue(Fact.objects.filter( desired_fact=self.desired_fact, data='01').exists()) self.assertTrue(Fact.objects.filter( desired_fact=self.desired_fact, data='02').exists())
def spurl(parser, token): bits = token.split_contents() if len(bits) < 2: raise TemplateSyntaxError("'spurl' takes at least one argument") kwargs = MultiValueDict() asvar = None bits = bits[1:] if len(bits) >= 2 and bits[-2] == 'as': asvar = bits[-1] bits = bits[:-2] for bit in bits: name, value = kwarg_re.match(bit).groups() if not (name and value): raise TemplateSyntaxError("Malformed arguments to spurl tag") kwargs.appendlist(name, parser.compile_filter(value)) return SpurlNode(kwargs, asvar)
def index(request): # Save current language in session (in case the user used a i18n url) lang = translation.get_language() request.session['django_language'] = lang date_min, date_max = DateRange.get_min_max() form = SearchForm(date_min, date_max, request.GET) if form.is_valid(): cd = form.cleaned_data first_time_message = not request.session.get('visited', False) request.session['visited'] = True workspec_search_url = reverse('api_dispatch_list', kwargs={ 'api_name': 'v1', 'resource_name': 'workspec_search'}) query = MultiValueDict() query['format'] = 'json' for lang in cd['languages']: query.appendlist('language__in', lang) for dom in cd['domains']: query.appendlist('activity_domain__in', dom) start_min = cd['start_min'] end_max = cd['end_max'] if start_min: query['start_min'] = start_min if end_max: query['end_max'] = end_max workspec_search_url += '?' + urlencode(query, doseq=True) address_url = reverse('api_dispatch_list', kwargs={ 'api_name': 'v1', 'resource_name': 'address'}) context = {'search_form' : form, 'first_time_message' : first_time_message, 'workspec_search_url' : workspec_search_url, 'address_url': address_url} return render(request, 'index.html', context) else: return redirect('index')
def _populate(self): # Short-circuit if called recursively in this thread to prevent # infinite recursion. Concurrent threads may call this at the same # time and will need to continue, so set 'populating' on a # thread-local variable. if getattr(self._local, 'populating', False): return try: self._local.populating = True lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for url_pattern in reversed(self.url_patterns): p_pattern = url_pattern.pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(url_pattern, URLPattern): self._callback_strs.add(url_pattern.lookup_str) bits = normalize(url_pattern.pattern.regex.pattern) lookups.appendlist( url_pattern.callback, (bits, p_pattern, url_pattern.default_args, url_pattern.pattern.converters) ) if url_pattern.name is not None: lookups.appendlist( url_pattern.name, (bits, p_pattern, url_pattern.default_args, url_pattern.pattern.converters) ) else: # url_pattern is a URLResolver. url_pattern._populate() if url_pattern.app_name: apps.setdefault(url_pattern.app_name, []).append(url_pattern.namespace) namespaces[url_pattern.namespace] = (p_pattern, url_pattern) else: for name in url_pattern.reverse_dict: for matches, pat, defaults, converters in url_pattern.reverse_dict.getlist(name): new_matches = normalize(p_pattern + pat) lookups.appendlist( name, ( new_matches, p_pattern + pat, {**defaults, **url_pattern.default_kwargs}, {**self.pattern.converters, **url_pattern.pattern.converters, **converters} ) ) for namespace, (prefix, sub_pattern) in url_pattern.namespace_dict.items(): current_converters = url_pattern.pattern.converters sub_pattern.pattern.converters.update(current_converters) namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in url_pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) self._callback_strs.update(url_pattern._callback_strs) self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._reverse_dict[language_code] = lookups self._populated = True finally: self._local.populating = False
def _populate(self): lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) lookups.appendlist(name, (new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs))) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps
def formfield_for_dbfield(self, db_field, **kwargs): request = kwargs.pop("request", None) # Add a select field of available commands if db_field.name == 'command': choices_dict = MultiValueDict() for command, app in get_commands().items(): choices_dict.appendlist(app, command) choices = [] for key in choices_dict.keys(): #if str(key).startswith('<'): # key = str(key) commands = choices_dict.getlist(key) commands.sort() choices.append([key, [[c,c] for c in commands]]) kwargs['widget'] = forms.widgets.Select(choices=choices) return db_field.formfield(**kwargs) kwargs['request'] = request return super(JobAdmin, self).formfield_for_dbfield(db_field, **kwargs)
class RegexURLResolver(object): def __init__(self, regex, urlconf_name, default_kwargs=None): # regex is a string representing a regular expression. # urlconf_name is a string representing the module containing urlconfs. self.regex = re.compile(regex, re.UNICODE) self.urlconf_name = urlconf_name self.callback = None self.default_kwargs = default_kwargs or {} self._reverse_dict = MultiValueDict() def __repr__(self): return '<%s %s %s>' % (self.__class__.__name__, self.urlconf_name, self.regex.pattern) def _get_reverse_dict(self): if not self._reverse_dict and hasattr(self.urlconf_module, 'urlpatterns'): try: for pattern in reversed(self.urlconf_module.urlpatterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) self._reverse_dict.appendlist(name, (new_matches, p_pattern + pat)) else: bits = normalize(p_pattern) self._reverse_dict.appendlist(pattern.callback, (bits, p_pattern)) self._reverse_dict.appendlist(pattern.name, (bits, p_pattern)) except: self._reverse_dict.clear() raise return self._reverse_dict reverse_dict = property(_get_reverse_dict) def resolve(self, path): tried = [] match = self.regex.search(path) if match: new_path = path[match.end():] for pattern in self.urlconf_module.urlpatterns: try: sub_match = pattern.resolve(new_path) except Resolver404, e: tried.extend([(pattern.regex.pattern + ' ' + t) for t in e.args[0]['tried']]) else: if sub_match: sub_match_dict = dict([(smart_str(k), v) for k, v in match.groupdict().items()]) sub_match_dict.update(self.default_kwargs) for k, v in sub_match[2].iteritems(): sub_match_dict[smart_str(k)] = v return sub_match[0], sub_match[1], sub_match_dict tried.append(pattern.regex.pattern) raise Resolver404, {'tried': tried, 'path': new_path}
def _populate(self): lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if hasattr(pattern, '_callback_str'): self._callback_strs.add(pattern._callback_str) elif hasattr(pattern, '_callback'): callback = pattern._callback if isinstance(callback, functools.partial): callback = callback.func if not hasattr(callback, '__name__'): lookup_str = callback.__module__ + "." + callback.__class__.__name__ else: lookup_str = callback.__module__ + "." + callback.__name__ self._callback_strs.add(lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist( name): new_matches = normalize(parent_pat + pat) lookups.appendlist( name, (new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs))) for namespace, ( prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True
def formfield_for_dbfield(self, db_field, **kwargs): request = kwargs.pop("request", None) # Add a select field of available commands if db_field.name == 'command': choices_dict = MultiValueDict() #l = get_commands().items(): #l = [('kitsune_base_check', 'kitsune')] l = get_kitsune_checks() for command, app in l: choices_dict.appendlist(app, command) choices = [] for key in choices_dict.keys(): #if str(key).startswith('<'): # key = str(key) commands = choices_dict.getlist(key) commands.sort() choices.append([key, [[c, c] for c in commands]]) kwargs['widget'] = forms.widgets.Select(choices=choices) return db_field.formfield(**kwargs) kwargs['request'] = request return super(JobAdmin, self).formfield_for_dbfield(db_field, **kwargs)
class RegexURLResolver(object): def __init__(self, regex, urlconf_name, default_kwargs=None): # regex is a string representing a regular expression. # urlconf_name is a string representing the module containing urlconfs. self.regex = re.compile(regex, re.UNICODE) self.urlconf_name = urlconf_name self.callback = None self.default_kwargs = default_kwargs or {} self._reverse_dict = MultiValueDict() def __repr__(self): return '<%s %s %s>' % (self.__class__.__name__, self.urlconf_name, self.regex.pattern) def _get_reverse_dict(self): if not self._reverse_dict and hasattr(self.urlconf_module, 'urlpatterns'): for pattern in reversed(self.urlconf_module.urlpatterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) self._reverse_dict.appendlist(name, (new_matches, p_pattern + pat)) else: bits = normalize(p_pattern) self._reverse_dict.appendlist(pattern.callback, (bits, p_pattern)) self._reverse_dict.appendlist(pattern.name, (bits, p_pattern)) return self._reverse_dict reverse_dict = property(_get_reverse_dict) def resolve(self, path): tried = [] match = self.regex.search(path) if match: new_path = path[match.end():] for pattern in self.urlconf_module.urlpatterns: try: sub_match = pattern.resolve(new_path) except Resolver404, e: tried.extend([(pattern.regex.pattern + ' ' + t) for t in e.args[0]['tried']]) else: if sub_match: sub_match_dict = dict([(smart_str(k), v) for k, v in match.groupdict().items()]) sub_match_dict.update(self.default_kwargs) for k, v in sub_match[2].iteritems(): sub_match_dict[smart_str(k)] = v return sub_match[0], sub_match[1], sub_match_dict tried.append(pattern.regex.pattern) raise Resolver404, {'tried': tried, 'path': new_path}
def _populate(self): if self._populating: return self._populating = True lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if isinstance(pattern, RegexURLPattern): self._callback_strs.add(pattern.lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist( name): new_matches = normalize(parent_pat + pat) lookups.appendlist(name, ( new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs), )) for namespace, ( prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) if not pattern._populating: pattern._populate() self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True self._populating = False
def _populate(self): # Short-circuit if called recursively in this thread to prevent # infinite recursion. Concurrent threads may call this at the same # time and will need to continue, so set 'populating' on a # thread-local variable. if getattr(self._local, 'populating', False): return self._local.populating = True lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if isinstance(pattern, RegexURLPattern): self._callback_strs.add(pattern.lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = normalize(parent_pat + pat) lookups.appendlist( name, ( new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs), ) ) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) if not getattr(pattern._local, 'populating', False): pattern._populate() self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True self._local.populating = False
def _populate(self): lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if hasattr(pattern, '_callback_str'): self._callback_strs.add(pattern._callback_str) elif hasattr(pattern, '_callback'): callback = pattern._callback if isinstance(callback, functools.partial): callback = callback.func if not hasattr(callback, '__name__'): lookup_str = callback.__module__ + "." + callback.__class__.__name__ else: lookup_str = callback.__module__ + "." + callback.__name__ self._callback_strs.add(lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = normalize(parent_pat + pat) lookups.appendlist( name, ( new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs), ) ) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True
def _populate(self): if self._populating: return self._populating = True lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if isinstance(pattern, RegexURLPattern): self._callback_strs.add(pattern.lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = normalize(parent_pat + pat) lookups.appendlist( name, ( new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs), ) ) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) if not pattern._populating: pattern._populate() self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True self._populating = False
def _populate(self): lookups = MultiValueDict() # key-list namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): # def url_patterns(self): 从模块中加载 urlpatterns # pattern 是 RegexURLPattern 类型 p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): # 如果就是本身类 RegexURLResolver 的一个实例 if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: # 设置为空 apps.setdefault(pattern.app_name, [] ).append(pattern.namespace) else: # 如果就是本身类 RegexURLResolver 的一个实例, 但不存在命名空间 parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) lookups.appendlist(name, (new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs))) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps
def _populate(self): # Short-circuit if called recursively in this thread to prevent # infinite recursion. Concurrent threads may call this at the same # time and will need to continue, so set 'populating' on a # thread-local variable. if getattr(self._local, "populating", False): return self._local.populating = True lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): if isinstance(pattern, RegexURLPattern): self._callback_strs.add(pattern.lookup_str) p_pattern = pattern.regex.pattern if p_pattern.startswith("^"): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent_pat = pattern.regex.pattern for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist(name): new_matches = normalize(parent_pat + pat) lookups.appendlist( name, (new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs)) ) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) if not getattr(pattern._local, "populating", False): pattern._populate() self._callback_strs.update(pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict[language_code] = lookups self._namespace_dict[language_code] = namespaces self._app_dict[language_code] = apps self._populated = True self._local.populating = False
def _populate(self): lookups = MultiValueDict() namespaces = {} apps = {} for pattern in reversed(self.url_patterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat, defaults in pattern.reverse_dict.getlist( name): new_matches = [] for piece, p_args in parent: new_matches.extend([ (piece + suffix, p_args + args) for (suffix, args) in matches ]) lookups.appendlist( name, (new_matches, p_pattern + pat, dict(defaults, **pattern.default_kwargs))) for namespace, ( prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern, pattern.default_args)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern, pattern.default_args)) self._reverse_dict = lookups self._namespace_dict = namespaces self._app_dict = apps
def _populate(self): # Almost the same as the original `_populate` function, except the last # 4 lines of code. lookups = MultiValueDict() namespaces = {} apps = {} language_code = get_language() for pattern in reversed(self.url_patterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([ (piece + suffix, p_args + args) for (suffix, args) in matches ]) lookups.appendlist(name, (new_matches, p_pattern + pat)) for namespace, ( prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern)) if pattern.name is not None: lookups.appendlist(pattern.name, (bits, p_pattern)) self._i18n_reverse_dict[language_code] = lookups self._i18n_namespace_dict[language_code] = namespaces self._i18n_app_dict[language_code] = apps
def _build_reverse_dict_for_lang(self, lang): reverse_dict = MultiValueDict() namespaces = {} apps = {} for pattern in reversed(self.url_patterns): if hasattr(pattern, 'get_regex'): p_pattern = pattern.get_regex(lang).pattern else: p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): if pattern.namespace: namespaces[pattern.namespace] = (p_pattern, pattern) if pattern.app_name: apps.setdefault(pattern.app_name, []).append(pattern.namespace) else: if hasattr(pattern, 'get_regex'): parent = normalize(pattern.get_regex(lang).pattern) else: parent = normalize(pattern.regex.pattern) if hasattr(pattern, 'get_reverse_dict'): sub_reverse_dict = pattern.get_reverse_dict(lang) else: sub_reverse_dict = pattern.reverse_dict for name in sub_reverse_dict: for matches, pat in sub_reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) reverse_dict.appendlist(name, (new_matches, p_pattern + pat)) for namespace, (prefix, sub_pattern) in pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) else: bits = normalize(p_pattern) reverse_dict.appendlist(pattern.callback, (bits, p_pattern)) reverse_dict.appendlist(pattern.name, (bits, p_pattern)) self._namespace_dict = namespaces self._app_dict = apps return reverse_dict
def _get_reverse_dict(self): if not self._reverse_dict: lookups = MultiValueDict() for pattern in reversed(self.url_patterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) lookups.appendlist(name, (new_matches, p_pattern + pat)) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern)) lookups.appendlist(pattern.name, (bits, p_pattern)) self._reverse_dict = lookups return self._reverse_dict
def _get_reverse_dict(self): if not self._reverse_dict and hasattr(self.urlconf_module, 'urlpatterns'): lookups = MultiValueDict() for pattern in reversed(self.urlconf_module.urlpatterns): p_pattern = pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(pattern, RegexURLResolver): parent = normalize(pattern.regex.pattern) for name in pattern.reverse_dict: for matches, pat in pattern.reverse_dict.getlist(name): new_matches = [] for piece, p_args in parent: new_matches.extend([(piece + suffix, p_args + args) for (suffix, args) in matches]) lookups.appendlist(name, (new_matches, p_pattern + pat)) else: bits = normalize(p_pattern) lookups.appendlist(pattern.callback, (bits, p_pattern)) lookups.appendlist(pattern.name, (bits, p_pattern)) self._reverse_dict = lookups return self._reverse_dict
def lxml_form_data(form): """Get form data as MultiValueDict from lxml Form element""" data = MultiValueDict() for obj in form.cssselect('input[type=text],' 'input[type=hidden],' 'textarea'): data.appendlist( obj.name, obj.value or '' ) for obj in form.cssselect('input[type=checkbox][checked],' 'input[type=radio][checked]'): data.appendlist( obj.name, obj.value or '', ) for obj in form.cssselect('select'): for opt in obj.cssselect('option[selected]'): data.appendlist( obj.name, opt.get('value'), ) return data
def appendlist(self, key, value): self._assert_mutable() key = str_to_unicode(key, self.encoding) value = str_to_unicode(value, self.encoding) MultiValueDict.appendlist(self, key, value)
def _populate(self): # Short-circuit if called recursively in this thread to prevent # infinite recursion. Concurrent threads may call this at the same # time and will need to continue, so set 'populating' on a # thread-local variable. #pylint:disable=protected-access,too-many-locals if getattr(self._local, 'populating', False): return try: self._local.populating = True lookups = MultiValueDict() namespaces = {} apps = {} path_prefix = self._get_path_prefix() for url_pattern in reversed(self.url_patterns): if isinstance(url_pattern, DjangoRegexURLPattern): self._callback_strs.add(url_pattern.lookup_str) # could be RegexURLPattern.regex or RegexURLResolver.regex here. p_pattern = url_pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(url_pattern, DjangoRegexURLResolver): if url_pattern.namespace: namespaces[url_pattern.namespace] = (p_pattern, url_pattern) if url_pattern.app_name: apps.setdefault(url_pattern.app_name, []).append(url_pattern.namespace) else: parent_pat = url_pattern.regex.pattern for name in url_pattern.reverse_dict: for _, pat, defaults \ in url_pattern.reverse_dict.getlist(name): new_matches = normalize(parent_pat + pat) lookups.appendlist(name, ( new_matches, p_pattern + pat, dict(defaults, ** url_pattern.default_kwargs), )) for namespace, (prefix, sub_pattern) \ in url_pattern.namespace_dict.items(): namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in \ url_pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) if not getattr(url_pattern._local, 'populating', False): url_pattern._populate() self._callback_strs.update(url_pattern._callback_strs) else: bits = normalize(p_pattern) lookups.appendlist( url_pattern.callback, (bits, p_pattern, url_pattern.default_args)) if url_pattern.name is not None: lookups.appendlist( url_pattern.name, (bits, p_pattern, url_pattern.default_args)) self._reverse_dict[path_prefix] = lookups self._namespace_dict[path_prefix] = namespaces self._app_dict[path_prefix] = apps self._populated = True finally: self._local.populating = False
class MultiPartParser(object): """ A rfc2388 multipart/form-data parser. ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ def __init__(self, META, input_data, upload_handlers, encoding=None): """ Initialize the MultiPartParser object. :META: The standard ``META`` dictionary in Django request objects. 字典 :input_data: The raw post data, as a file-like object. post 数据 :upload_handler: An UploadHandler instance that performs operations on the uploaded data. :encoding: The encoding with which to treat the incoming data. """ # # Content-Type should containt multipart and the boundary information. # content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', '')) if not content_type.startswith('multipart/'): raise MultiPartParserError('Invalid Content-Type: %s' % content_type) # invalid content-type multipart 是无效的 # Parse the header to get the boundary to split the parts. boundary 界限, 将数据分成就给部分 # parse_header 可以分析 HTTP 头的某一个部位 ctypes, opts = parse_header(content_type.encode('ascii')) boundary = opts.get('boundary') if not boundary or not cgi.valid_boundary(boundary): raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary) # Content-Length should contain the length of the body we are about # to receive. try: content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH', 0))) except (ValueError, TypeError): content_length = 0 if content_length < 0: # This means we shouldn't continue...raise an error. raise MultiPartParserError("Invalid content length: %r" % content_length) if isinstance(boundary, six.text_type): # 转化为 ascii boundary = boundary.encode('ascii') self._boundary = boundary self._input_data = input_data # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] self._chunk_size = min([2**31-4] + possible_sizes) self._meta = META #正常赋值 self._encoding = encoding or settings.DEFAULT_CHARSET# 如果没有,设定 setting 中的值 self._content_length = content_length # 上面获取 self._upload_handlers = upload_handlers # 上面获取 def parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. Returns a tuple containing the POST and FILES dictionary, respectively. """ # We have to import QueryDict down here to avoid a circular import. from django.http import QueryDict encoding = self._encoding handlers = self._upload_handlers # HTTP spec says that Content-Length >= 0 is valid # handling content-length == 0 before continuing if self._content_length == 0: return QueryDict('', encoding=self._encoding), MultiValueDict() # See if the handler will want to take care of the parsing. # This allows overriding everything if somebody wants it. for handler in handlers: result = handler.handle_raw_input(self._input_data, self._meta, self._content_length, self._boundary, encoding) if result is not None: return result[0], result[1] # 返回 # Create the data structures to be used later. self._post = QueryDict('', mutable=True) self._files = MultiValueDict() # Instantiate the parser and stream: stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) # 未知 # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None counters = [0] * len(handlers) try: for item_type, meta_data, field_stream in Parser(stream, self._boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. self.handle_file_complete(old_field_name, counters) 这里更新了 files old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') 不懂 if item_type == FIELD: # This is a post field, post 域 we can just set it in the post if transfer_encoding == 'base64': raw_data = field_stream.read() 开始读 try: data = str(raw_data).decode('base64') except: data = raw_data else: data = field_stream.read() self._post.appendlist(field_name, force_text(data, encoding, errors='replace')) elif item_type == FILE: # This is a file, use the handler... 是一个文件 file_name = disposition.get('filename') if not file_name: continue file_name = force_text(file_name, encoding, errors='replace') 不懂 file_name = self.IE_sanitize(unescape_entities(file_name)) 跟 IE 有关 content_type = meta_data.get('content-type', ('',))[0].strip() try: charset = meta_data.get('content-type', (0, {}))[1].get('charset', None) except: charset = None try: content_length = int(meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): content_length = None counters = [0] * len(handlers) try: for handler in handlers: try: handler.new_file(field_name, file_name, content_type, content_length, charset) except StopFutureHandlers: break for chunk in field_stream: if transfer_encoding == 'base64': # We only special-case base64 transfer encoding # We should always read base64 streams by multiple of 4 over_bytes = len(chunk) % 4 if over_bytes: over_chunk = field_stream.read(4 - over_bytes) chunk += over_chunk try: chunk = base64.b64decode(chunk) except Exception as e: # Since this is only a chunk, any error is an unfixable error. raise MultiPartParserError("Could not decode base64 data: %r" % e) for i, handler in enumerate(handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: # If the chunk received by the handler is None, then don't continue. break except SkipFile: # Just use up the rest of this file... exhaust(field_stream) else: # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) except StopUpload as e: if not e.connection_reset: exhaust(self._input_data) else: # Make sure that the request data is all fed exhaust(self._input_data) # Signal that the upload has completed. for handler in handlers: retval = handler.upload_complete() if retval: break return self._post, self._files # 返回 post 数据和 file 处理文件的方法, 在这里会更新 _files def handle_file_complete(self, old_field_name, counters): """ Handle all the signalling that takes place when a file is complete. """ for i, handler in enumerate(self._upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: # If it returns a file object, then set the files dict. self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj) break def IE_sanitize(self, filename): """Cleanup filename from Internet Explorer full paths.""" return filename and filename[filename.rfind("\\")+1:].strip() 可能只需要文件名,而不用其他的
def prepare_base_review_request_mail(user, review_request, subject, in_reply_to, to_field, cc_field, template_name_base, context=None, extra_headers=None): """Return a customized review request e-mail. This is intended to be called by one of the ``prepare_{type}_mail`` functions in this file. This method builds up a common context that all review request-related e-mails will use to render their templates, as well as handling user preferences regarding e-mail and add adding additional headers. Args: user (django.contrib.auth.models.User): The user who is sending the e-mail. review_request (reviewboard.reviews.models.review_request.ReviewRequest): The review request this e-mail is regarding. subject (unicode): The e-mail subject line. in_reply_to (unicode): The e-mail message ID this message is in response to or ``None``. to_field (set): The set of :py:class:`~django.contrib.auth.models.User` and :py:class`~reviewboard.reviews.models.group.Group`s to this e-mail will be sent to. cc_field (set): The set of :py:class:`~django.contrib.auth.models.User` and :py:class`~reviewboard.reviews.models.group.Group`s to be CC'ed on the e-mail. template_name_base (unicode): The name of the template to use to generate the e-mail without its extension. The plain-text version of the e-mail will append ``.txt`` to this and and the rich-text version of the e-mail will append ``.html``. context (dict, optional): Optional additional template rendering context. extra_headers (dict, optional): Optional additional headers to include. Returns: EmailMessage: The prepared e-mail message. """ user_email = build_email_address_for_user(user) to_field = recipients_to_addresses(to_field, review_request.id) cc_field = recipients_to_addresses(cc_field, review_request.id) - to_field if not user.should_send_own_updates(): to_field.discard(user_email) cc_field.discard(user_email) if not to_field and not cc_field: # This e-mail would have no recipients, so we won't send it. return None if not context: context = {} context.update({ 'user': user, 'site_url': get_server_url(), 'review_request': review_request, }) local_site = review_request.local_site if local_site: context['local_site_name'] = local_site.name text_body = render_to_string('%s.txt' % template_name_base, context) html_body = render_to_string('%s.html' % template_name_base, context) server_url = get_server_url(local_site=local_site) headers = MultiValueDict({ 'X-ReviewBoard-URL': [server_url], 'X-ReviewRequest-URL': [ build_server_url(review_request.get_absolute_url(), local_site=local_site) ], 'X-ReviewGroup': [ ', '.join( review_request.target_groups.values_list('name', flat=True)) ], }) if extra_headers: if not isinstance(extra_headers, MultiValueDict): extra_headers = MultiValueDict( (key, [value]) for key, value in six.iteritems(extra_headers)) headers.update(extra_headers) if review_request.repository: headers['X-ReviewRequest-Repository'] = review_request.repository.name latest_diffset = review_request.get_latest_diffset() if latest_diffset: modified_files = set() for filediff in latest_diffset.files.all(): if filediff.deleted or filediff.copied or filediff.moved: modified_files.add(filediff.source_file) if filediff.is_new or filediff.copied or filediff.moved: modified_files.add(filediff.dest_file) # The following code segment deals with the case where the client adds # a significant amount of files with large names. We limit the number # of headers; when more than 8192 characters are reached, we stop # adding filename headers. current_header_length = 0 for filename in modified_files: current_header_length += (HEADER_ADDITIONAL_CHARACTERS_LENGTH + len(filename)) if current_header_length > MAX_FILENAME_HEADERS_LENGTH: logging.warning( 'Unable to store all filenames in the ' 'X-ReviewBoard-Diff-For headers when sending e-mail for ' 'review request %s: The header size exceeds the limit of ' '%s. Remaining headers have been omitted.', review_request.display_id, MAX_FILENAME_HEADERS_LENGTH) break headers.appendlist('X-ReviewBoard-Diff-For', filename) if settings.DEFAULT_FROM_EMAIL: sender = build_email_address(full_name=user.get_full_name(), email=settings.DEFAULT_FROM_EMAIL) else: sender = None return EmailMessage(subject=subject.strip(), text_body=text_body.encode('utf-8'), html_body=html_body.encode('utf-8'), from_email=user_email, sender=sender, to=list(to_field), cc=list(cc_field), in_reply_to=in_reply_to, headers=headers)
def _handle_raw_input_without_file_stream(self, input_data, META, raw_content_length, boundary, encoding=None): """ Replaces django.http.multipartparser.MultiPartParser.parse A rfc2388 multipart/form-data parser but replacing the file stream to the creation of empty files. Returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ # Create the data structures to be used later. _post = QueryDict(mutable=True) _files = MultiValueDict() # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. _chunk_size = min([2**31 - 4, self.chunk_size]) # Instantiate the parser and stream: stream = LazyStream(ChunkIter(input_data, _chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None # Number of bytes that have been read. num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 # To limit the amount of data read from the request. read_size = None for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. file_obj = self.file_complete(raw_content_length) if file_obj: # If it returns a file object, then set the files dict. _files.appendlist( force_str(old_field_name, encoding, errors="replace"), file_obj) old_field_name = None try: disposition = meta_data["content-disposition"][1] field_name = disposition["name"].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get("content-transfer-encoding") if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_str(field_name, encoding, errors="replace") if item_type == FIELD: # NOTE: Parse fields as usual, same as ``MultiPartParser.parse`` # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. num_post_keys += 1 if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): raise TooManyFieldsSent( "The number of GET/POST parameters exceeded " "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.") # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read # This is a post field, we can just set it in the post if transfer_encoding == "base64": raw_data = field_stream.read(size=read_size) num_bytes_read += len(raw_data) try: data = base64.b64decode(raw_data) except binascii.Error: data = raw_data else: data = field_stream.read(size=read_size) num_bytes_read += len(data) # Add two here to make the check consistent with the # x-www-form-urlencoded check that includes '&='. num_bytes_read += len(field_name) + 2 if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE): raise RequestDataTooBig( "Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE." ) _post.appendlist(field_name, force_str(data, encoding, errors="replace")) elif item_type == FILE: # NOTE: Parse files WITHOUT a stream. # This is a file, use the handler... file_name = disposition.get("filename") if file_name: file_name = force_str(file_name, encoding, errors="replace") file_name = self.sanitize_file_name(file_name) if not file_name: continue content_type, content_type_extra = meta_data.get( "content-type", ("", {})) content_type = content_type.strip() charset = content_type_extra.get("charset") content_length = None self.new_file(field_name, file_name, content_type, content_length, charset, content_type_extra) # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) # Make sure that the request data is all fed exhaust(input_data) _post._mutable = False return _post, _files
class EmailBaseModel(models.Model): sender = models.EmailField(_("sender"), max_length=255) from_str = models.CharField(_("from"), max_length=255) recipient = models.CharField(_("recipient"), max_length=255) subject = models.CharField(_("subject"), max_length=255, blank=True) body_plain = models.TextField(_("body plain"), blank=True) body_html = models.TextField(_("body html"), blank=True) stripped_text = models.TextField(_("stripped text"), blank=True) stripped_html = models.TextField(_("stripped html"), blank=True) stripped_signature = models.TextField(_("stripped signature"), blank=True) message_headers = models.TextField(_("message headers"), blank=True, help_text=_("Stored in JSON.")) content_id_map = models.TextField(_("Content-ID map"), blank=True, help_text=_("Dictionary mapping Content-ID (CID) values to corresponding attachments. Stored in JSON.")) received = models.DateTimeField(_("received"), auto_now_add=True) class Meta: abstract = True verbose_name = _("incoming email") verbose_name_plural = _("incoming emails") def __init__(self, *args, **kwargs): super(EmailBaseModel, self).__init__(*args, **kwargs) self._headers = None self._cids = None def _load_headers(self): self._headers = MultiValueDict() try: header_list = json.loads(self.message_headers) for key, val in header_list: self._headers.appendlist(key, val) except: logger.exception("Error parsing JSON data containing message headers") @property def headers(self): """Access message_headers parsed into MultiValueDict""" if self._headers is None: self._load_headers() return self._headers def _load_cids(self): if self.content_id_map: self._cids = {} try: self._cids = json.loads(self.content_id_map) except: logger.exception("Error parsing JSON data containing Content-IDs") @property def content_ids(self): """Access content_id_map as dict""" if not self.content_id_map: return if self._cids is None: self._load_cids() return self._cids @property def message_id(self): return self.headers.get('Message-Id', None) @property def cc(self): return self.headers.get('Cc', None) @property def references(self): return self.headers.get('References', None) @property def in_reply_to(self): return self.headers.get('In-Reply-To', None) @property def site_url(self): return 'https://%s%s' % (Site.objects.get_current().domain, reverse('comlink:mail', kwargs={'id': self.id})) def get_mailgun_data(self, stripped=True, footer=True): if stripped: body_plain = self.stripped_text body_html = self.stripped_html else: body_plain = self.body_plain body_html = self.body_html if footer: # Add in a footer text_footer = "\n\n-------------------------------------------\n*~*~*~* Sent through Nadine *~*~*~*\n%s" % self.site_url body_plain = body_plain + text_footer if body_html: html_footer = "<br><br>-------------------------------------------<br>*~*~*~* Sent through Nadine *~*~*~*\n%s" % self.site_url body_html = body_html + html_footer # Build and return our data mailgun_data = {"from": self.from_str, "to": [self.recipient, ], "cc": [self.cc, ], "subject": self.subject, "text": body_plain, "html": body_html, } return mailgun_data def __str__(self): return _("Message from {from_str}: {subject_trunc}").format( from_str=self.from_str, subject_trunc=self.subject[:20])
def search(self, query_string, sort_by=None, start_offset=0, end_offset=None, fields='', highlight=False, facets=None, date_facets=None, query_facets=None, narrow_queries=None, spelling_query=None, limit_to_registered_models=None, debug=False, **kwargs): hits = 0 results = list() if query_string in (None, ''): return { 'results': results, 'hits': len(results), } #build search criteria qs = SearchableObject.objects.all() extras = MultiValueDict() if narrow_queries: models = dict() for query in narrow_queries: #TODO properly parse narrow queries if ':' in query: key, value = query.split(':', 1) if key == 'django_ct': if value not in models: app_label, model = value.split('.', 1) ct = ContentType.objects.get(app_label=app_label, model=model) models[value] = ct else: query_string[key] = value if models: limit_to_registered_models = models.values() if limit_to_registered_models is not None: qs = qs.filter(content_type__in=limit_to_registered_models) for key, value_list in query_string.iterlists(): if key == 'content': search_params = ' '.join(query_string.getlist('content')) qs &= SearchableObject.objects.search(search_params) else: #TODO support multi value search extras.appendlist('where', self.get_subquery_sql()) extras.appendlist('params', key) extras.appendlist('params', str(value_list[0])) if extras: qs = qs.extra(**extras) qs = qs.distinct() #may or may not work.... results = SearchObjectQuerySet(model=qs.model, query=qs.query) if debug: print len(results) return { 'results': results, 'hits': len(results), }
def test_appendlist(self): d = MultiValueDict() d.appendlist('name', 'Adrian') d.appendlist('name', 'Simon') self.assertEqual(d.getlist('name'), ['Adrian', 'Simon'])
def test_appendlist(self): d = MultiValueDict() d.appendlist("name", "Adrian") d.appendlist("name", "Simon") self.assertEqual(d.getlist("name"), ["Adrian", "Simon"])
def _populate(self): # Short-circuit if called recursively in this thread to prevent # infinite recursion. Concurrent threads may call this at the same # time and will need to continue, so set 'populating' on a # thread-local variable. if getattr(self._local, 'populating', False): return try: self._local.populating = True lookups = MultiValueDict() namespaces = {} apps = {} path_prefix = self._get_path_prefix() for url_pattern in reversed(self.url_patterns): p_pattern = url_pattern.pattern.regex.pattern if p_pattern.startswith('^'): p_pattern = p_pattern[1:] if isinstance(url_pattern, URLPattern): self._callback_strs.add(url_pattern.lookup_str) bits = normalize(url_pattern.pattern.regex.pattern) lookups.appendlist( url_pattern.callback, (bits, p_pattern, url_pattern.default_args, url_pattern.pattern.converters)) if url_pattern.name is not None: lookups.appendlist( url_pattern.name, (bits, p_pattern, url_pattern.default_args, url_pattern.pattern.converters)) else: # url_pattern is a URLResolver. url_pattern._populate() if url_pattern.app_name: apps.setdefault(url_pattern.app_name, []).append(url_pattern.namespace) namespaces[url_pattern.namespace] = (p_pattern, url_pattern) else: for name in url_pattern.reverse_dict: for matches, pat, defaults, converters in \ url_pattern.reverse_dict.getlist(name): new_matches = normalize(p_pattern + pat) lookups.appendlist( name, (new_matches, p_pattern + pat, { **defaults, **url_pattern.default_kwargs }, { **self.pattern.converters, **url_pattern.pattern.converters, **converters })) for namespace, (prefix, sub_pattern) in \ url_pattern.namespace_dict.items(): current_converters = \ url_pattern.pattern.converters sub_pattern.pattern.converters.update( current_converters) namespaces[namespace] = (p_pattern + prefix, sub_pattern) for app_name, namespace_list in \ url_pattern.app_dict.items(): apps.setdefault(app_name, []).extend(namespace_list) self._callback_strs.update(url_pattern._callback_strs) self._namespace_dict[path_prefix] = namespaces self._app_dict[path_prefix] = apps self._reverse_dict[path_prefix] = lookups self._populated = True finally: self._local.populating = False
def appendlist(self, key, value): self._assert_mutable() MultiValueDict.appendlist(self, key, value)
class ModelTree(object): """A class to handle building and parsing a tree structure given a model. `root_model` - The root or "reference" model for the tree. Everything is relative to the root model. `exclude` - A list of models that are to be excluded from this tree. This is typically used to exclude models not intended to be exposed through this API. `routes` - Explicitly defines a join path between two models. Each route is made up of four components. Assuming some model hierarchy exists as shown below.. ModelA / \ ModelB ModelC | | \ ModelD \ / ModelE ..the traversal path from ModelA to ModelE is ambiguous. It could go from A -> B -> E or A -> C -> D -> E. By default, the shortest path is always choosen to reduce the number of joins necessary, but if ModelD did not exist.. ModelA / \ ModelB ModelC \ / ModelE ..both paths only require two joins, thus the path that gets traversed first will only be the choosen one. To explicitly choose a path, a route can be defined. Taking the form:: { 'source': 'app1.model1', 'target': 'app1.model2', 'field': None, 'symmetrical': None, } The `source` model defines the model where the join is being created from (the left side of the join). The `target` model defines the target model (the right side of the join). `field` is optional, but explicitly defines the model field that will be used for the join. This is useful if there are more than one foreign key relationships on between target and source. Finally, `symmetrical` is an optional boolean that ensures when the target and source models switch sides, the same join occurs on the same field. Routes are typically used for defining explicit join paths, but sometimes it is necessary to exclude join paths. For example if there are three possible paths and one should never occur. A modeltree config can have `required_routes` and `excluded_routes` entries, which are lists of routes in the above format. A required route is defined as follows: a join to the specified target model is only allowed from the specified source model. A model can only be specified as a target once in the list of required routes. Note that the use of the `symmetrical` property of a route implicitly adds another route with target and source models swapped, so a model can be a target either directly or indirectly. A single source model can participate in multiple required routes. An excluded route is more obvious: joining from the specified source model to the specified target model is not allowed. """ # noqa: W605 def __init__(self, model=None, **kwargs): if model is None and 'root_model' in kwargs: warnings.warn('The "root_model" key has been renamed to "model"', DeprecationWarning) model = kwargs.get('root_model') if not model: raise TypeError('No "model" defined') excluded_models = kwargs.get('excluded_models', ()) required_routes = kwargs.get('required_routes') if not excluded_models and 'exclude' in kwargs: warnings.warn( 'The "exclude" key has been renamed to ' '"excluded_models"', DeprecationWarning) excluded_models = kwargs.get('exclude', ()) if not required_routes and 'routes' in kwargs: warnings.warn( 'The "routes" key has been renamed to ' '"required_routes"', DeprecationWarning) required_routes = kwargs.get('routes') excluded_routes = kwargs.get('excluded_routes') self.root_model = self.get_model(model, local=False) self.alias = kwargs.get('alias', None) # Models completely excluded from the tree self.excluded_models = [ self.get_model(label, local=False) for label in excluded_models ] # Build the routes that are allowed/preferred self._required_joins = self._build_routes( required_routes, allow_redundant_targets=False) # Build the routes that are excluded self._excluded_joins = self._build_routes(excluded_routes) # cache each node relative their models self._nodes = {} # cache all app names relative to their model names i.e. supporting # multiple apps with models of the same name self._model_apps = MultiValueDict({}) # cache (app, model) pairs with the respective model class self._models = {} self._build() def __repr__(self): return '<ModelTree for {0}>'.format(self.root_model.__name__) def _get_local_model(self, model_name, app_name=None): "Attempts to get a model from local cache." if not app_name: app_names = self._model_apps.getlist(model_name) # No apps found with this model if not app_names: return # Multiple apps found for this model if len(app_names) > 1: raise ModelNotUnique( 'The model "{0}" is not unique. ' 'Specify the app name as well.'.format(model_name)) app_name = app_names[0] try: return self._models[(app_name, model_name)] except KeyError: pass def _get_model(self, model_name, app_name=None): "Attempts to get a model from application cache." model = None # If an app name is supplied we can reduce it down to only models # within that particular app. if app_name: model = apps.get_model(app_name, model_name) else: # Attempt to find the model based on the name. Since we don't # have the app name, if a model of the same name exists multiple # times, we need to throw an error. for app, app_models in list(apps.app_models.items()): if model_name in app_models: if model is not None: raise ModelNotUnique( 'The model "{0}" is not unique. ' 'Specify the app name as well.'.format(model_name)) model = app_models[model_name] return model def get_model(self, model_name=None, app_name=None, local=True): """A few variations are handled here for increased flexibility: - if a model class is given, simply echo the model back - if a app-model label e.g. 'library.book', is passed, the standard app_models cache is used - if `app_name` and `model_name` is provided, the standard app_models cache is used - if only `model_name` is supplied, attempt to find the model across all apps. if the model is found more than once, an error is thrown - if `local` is true, only models related to this `ModelTree` instance are searched through """ model = None if not (app_name or model_name): return self.root_model # model class if inspect.isclass(model_name) and \ issubclass(model_name, models.Model): # set it initially for either local and non-local model = model_name # additional check to ensure the model exists locally, reset to # None if it does not if local and model not in self._nodes: model = None # handle string-based arguments else: # handle the syntax 'library.book' if model_name: if '.' in model_name: app_name, model_name = model_name.split('.', 1) model_name = model_name.lower() if local: model = self._get_local_model(model_name, app_name) else: model = self._get_model(model_name, app_name) # both mechanisms above may result in no model being found if model is None: if local: raise ModelNotRelated( 'No model found named "{0}"'.format(model_name)) else: raise ModelDoesNotExist( 'No model found named "{0}"'.format(model_name)) return model def get_field(self, name, model=None): if model is None: model = self.root_model return model._meta.get_field(name) def _build_routes(self, routes, allow_redundant_targets=True): """Routes provide a means of specifying JOINs between two tables. routes - a collection of dicts defining source->target mappings with optional `field` specifier and `symmetrical` attribute. allow_redundant_targets - whether two routes in this collection are allowed to have the same target - this should NOT be allowed for required routes. """ routes = routes or () joins = {} targets_seen = set() for route in routes: if isinstance(route, dict): source_label = route.get('source') target_label = route.get('target') field_label = route.get('field') symmetrical = route.get('symmetrical') else: warnings.warn('Routes are now defined as dicts', DeprecationWarning) source_label, target_label, field_label, symmetrical = route # get models source = self.get_model(source_label, local=False) target = self.get_model(target_label, local=False) field = None # get field if field_label: model_name, field_name = field_label.split('.', 1) model_name = model_name.lower() # determine which model the join field specified exists on if model_name == source.__name__.lower(): field = self.get_field(field_name, source) elif model_name == target.__name__.lower(): field = self.get_field(field_name, target) else: raise TypeError('model for join field, "{0}", ' 'does not exist'.format(field_name)) if isinstance(field, (ManyToOneRel, ManyToManyRel)): field = field.field if not allow_redundant_targets: if target in targets_seen: raise ValueError('Model {0} cannot be the target of ' 'more than one route in this list'.format( target_label)) else: targets_seen.add(target) # The `joins` hash defines pairs which are explicitly joined # via the specified field. If no field is defined, then the # join field is implied or does not matter; the route is reduced # to a straight lookup. joins[(source, target)] = field if symmetrical: if not allow_redundant_targets: if source in targets_seen: raise ValueError( 'Model {0} cannot be the target of ' 'more than one route in this list'.format( source_label)) else: targets_seen.add(source) joins[(target, source)] = field return joins def _join_allowed(self, source, target, field=None): """Checks if the join between `source` and `target` via `field` is allowed. """ join = (source, target) # No circles if target == source: return False # Prevent join to excluded models if target in self.excluded_models: return False # Never go back through the root if target == self.root_model: return False # Apply excluded joins if any if join in self._excluded_joins: _field = self._excluded_joins[join] if not _field: return False elif _field and _field == field: return False # Check if the join is allowed by a required rule for (_source, _target), _field in list(self._required_joins.items()): if _target == target: if _source != source: return False # If a field is supplied, check to see if the field is allowed # for this join. if field and _field and _field != field: return False return True def _add_node(self, parent, model, relation, reverse, related_name, accessor_name, nullable, depth): """Adds a node to the tree only if a node of the same `model' does not already exist in the tree with smaller depth. If the node is added, the tree traversal continues finding the node's relations. Conditions in which the node will fail to be added: - a reverse relationship is blocked via the '+' - the model is excluded completely - the model is going back the same path it came from - the model is circling back to the root_model - the model does not come from an explicitly declared parent model This is running in a recursive way with _find_relations They keep calling each other as they go through all the models """ # Reverse relationships if reverse and '+' in related_name: return node_hash = self._nodes.get(model, None) # don't add node if a path with a shorter depth exists. this is applied # after the correct join has been determined. generally if a route is # defined for relation, this will never be an issue since there would # only be one path available. if a route is not defined, the shorter # path will be found if not node_hash or node_hash['depth'] > depth: if node_hash: node_hash['parent'].remove_child(model) node = ModelTreeNode(model, parent, relation, reverse, related_name, accessor_name, nullable, depth) self._nodes[model] = { 'parent': parent, 'depth': depth, 'node': node, } node = self._find_relations(node, depth) parent.children.append(node) def _find_relations(self, node, depth=0): """ Finds all relations given a node. This runs in a recursive way with _add_node. They keep calling each other based on depth. """ depth += 1 model = node.model # NOTE: the many-to-many relations are evaluated first to prevent # 'through' models being bound as a ForeignKey relationship. fields = sorted(model._meta.get_fields(), reverse=True, key=lambda f: bool(f.many_to_many)) # determine relational fields to determine paths # f.rel changed to f.remote_field for django2 # f.rel.to changed to f.remote_field.model for django2 forward_fields = [ f for f in fields if (f.one_to_one or f.many_to_many or f.many_to_one) and ( f.concrete or not f.auto_created) and f.remote_field is not None # Generic foreign keys do not define rel. and self._join_allowed(f.model, f.remote_field.model, f) ] reverse_fields = [ f for f in fields if (f.one_to_many or f.one_to_one or f.many_to_many) and ( not f.concrete and f.auto_created) and self._join_allowed(f.model, f.related_model, f.field) ] def get_relation_type(f): if f.one_to_one: return 'onetone' elif f.many_to_many: return 'manytomany' elif f.one_to_many or f.many_to_one: return 'foreignkey' # Iterate over forward relations # changed f.rel.to to f.remote_field.model for django2 for f in forward_fields: null = f.many_to_many or f.null kwargs = { 'parent': node, 'model': f.remote_field.model, 'relation': get_relation_type(f), 'reverse': False, 'related_name': f.name, 'accessor_name': f.name, 'nullable': null, 'depth': depth, } self._add_node(**kwargs) # Iterate over reverse relations. for r in reverse_fields: kwargs = { 'parent': node, 'model': r.related_model, 'relation': get_relation_type(r), 'reverse': True, 'related_name': r.field.related_query_name(), 'accessor_name': r.get_accessor_name(), 'nullable': True, 'depth': depth, } self._add_node(**kwargs) return node def _build(self): node = ModelTreeNode(self.root_model) self._root_node = self._find_relations(node) self._nodes[self.root_model] = { 'parent': None, 'depth': 0, 'node': self._root_node, } # store local cache of all models in this tree by name for model in self._nodes: model_name = model._meta.object_name.lower() app_name = model._meta.app_label self._model_apps.appendlist(model_name, app_name) self._models[(app_name, model_name)] = model @property def root_node(self): "Returns the `root_node` and implicitly builds the tree." if not hasattr(self, '_root_node'): self._build() return self._root_node def _node_path_to_model(self, model, node, path=[]): "Returns a list representing the path of nodes to the model." if node.model == model: return path for child in node.children: mpath = self._node_path_to_model(model, child, path + [child]) # TODO why is this condition here? if mpath: return mpath def _node_path(self, model): "Returns a list of nodes thats defines the path of traversal." model = self.get_model(model) return self._node_path_to_model(model, self.root_node) def get_joins(self, model): """Returns a list of JOIN connections that can be manually applied to a QuerySet object. See `.add_joins()` This allows for the ORM to handle setting up the JOINs which may be different depending on the QuerySet being altered. """ node_path = self._node_path(model) joins = [] for i, node in enumerate(node_path): # ignore each subsequent first join in the set of joins for a # given model table, path_joins = node.get_joins() if i == 0: joins.append(table) joins.extend(path_joins) return joins def query_string(self, model): nodes = self._node_path(model) return str('__'.join(n.related_name for n in nodes)) def query_string_for_field(self, field, operator=None, model=None): """Takes a `models.Field` instance and returns a query string relative to the root model. """ if model: if model._meta.proxy and \ model._meta.proxy_for_model is not field.model: raise ModelTreeError('proxied model must be the field model') else: model = field.model # When an explicit reverse field is used, simply use it directly if isinstance(field, (ManyToManyRel, ManyToOneRel)): toks = [field.field.related_query_name()] else: path = self.query_string(model) if path: toks = [path, field.name] else: toks = [field.name] if operator is not None: toks.append(operator) return str('__'.join(toks)) def query_condition(self, field, operator, value, model=None): "Conveniece method for constructing a `Q` object for a given field." lookup = self.query_string_for_field(field, operator=operator, model=model) return Q(**{lookup: value}) def add_joins(self, model, queryset=None): """Sets up all necessary joins up to the given model on the queryset. Returns the alias to the model's database table. """ if queryset is None: clone = self.get_queryset() else: clone = queryset._clone() alias = None for i, join in enumerate(self.get_joins(model)): if isinstance(join, BaseTable): alias_map = clone.query.alias_map if join.table_alias in alias_map or \ join.table_name in alias_map: continue alias = clone.query.join(join) # this implies the join is redundant and occurring on the root model's # table if alias is None: alias = clone.query.get_initial_alias() return clone, alias def add_select(self, *fields, **kwargs): "Replaces the `SELECT` columns with the ones provided." if 'queryset' in kwargs: queryset = kwargs.pop('queryset') else: queryset = self.get_queryset() queryset.query.default_cols = False include_pk = kwargs.pop('include_pk', True) if include_pk: fields = [self.root_model._meta.pk] + list(fields) aliases = [] for pair in fields: if isinstance(pair, (list, tuple)): model, field = pair else: field = pair model = field.model queryset, alias = self.add_joins(model, queryset) aliases.append(Col(alias, field, field)) if aliases: queryset.query.select = aliases return queryset def get_queryset(self): "Returns a QuerySet relative to the `root_model`." return self.root_model._default_manager.get_queryset()
def send_review_mail(user, review_request, subject, in_reply_to, to_field, cc_field, text_template_name, html_template_name, context=None, extra_headers=None): """Format and send an e-mail out. Args: user (django.contrib.auth.models.User): The user who is sending the e-mail. review_request (reviewboard.reviews.models.ReviewRequest): The review request that the e-mail is about. subject (unicode): The subject of the e-mail address. in_reply_to (unicode): The e-mail message ID for threading. to_field (list): The recipients to send the e-mail to. This should be a list of :py:class:`Users <django.contrib.auth.models.User>` and :py:class:`Groups <reviewboard.reviews.models.Group>`. cc_field (list): The addresses to be CC'ed on the e-mail. This should be a list of :py:class:`Users <django.contrib.auth.models.User>` and :py:class:`Groups <reviewboard.reviews.models.Group>`. text_template_name (unicode): The name for the text e-mail template. html_template_name (unicode): The name for the HTML e-mail template. context (dict): Optional extra context to provide to the template. extra_headers (dict): Either a dict or :py:class:`~django.utils.datastructures.MultiValueDict` providing additional headers to send with the e-mail. Returns: unicode: The resulting e-mail message ID. """ current_site = Site.objects.get_current() local_site = review_request.local_site from_email = build_email_address_for_user(user) to_field = recipients_to_addresses(to_field, review_request.id) cc_field = recipients_to_addresses(cc_field, review_request.id) - to_field if not user.should_send_own_updates(): user_email = build_email_address_for_user(user) to_field.discard(user_email) cc_field.discard(user_email) if not to_field and not cc_field: # Nothing to send. return siteconfig = current_site.config.get() domain_method = siteconfig.get("site_domain_method") if not context: context = {} context['user'] = user context['domain'] = current_site.domain context['domain_method'] = domain_method context['review_request'] = review_request if review_request.local_site: context['local_site_name'] = review_request.local_site.name text_body = render_to_string(text_template_name, context) html_body = render_to_string(html_template_name, context) base_url = get_server_url(local_site=local_site) headers = MultiValueDict({ 'X-ReviewBoard-URL': [base_url], 'X-ReviewRequest-URL': [urljoin(base_url, review_request.get_absolute_url())], 'X-ReviewGroup': [ ', '.join(group.name for group in review_request.target_groups.all()) ], }) if extra_headers: if not isinstance(extra_headers, MultiValueDict): extra_headers = MultiValueDict( (key, [value]) for (key, value) in six.iteritems(extra_headers)) headers.update(extra_headers) if review_request.repository: headers['X-ReviewRequest-Repository'] = review_request.repository.name latest_diffset = review_request.get_latest_diffset() if latest_diffset: modified_files = set() for filediff in latest_diffset.files.all(): if filediff.deleted or filediff.copied or filediff.moved: modified_files.add(filediff.source_file) if filediff.is_new or filediff.copied or filediff.moved: modified_files.add(filediff.dest_file) for filename in modified_files: headers.appendlist('X-ReviewBoard-Diff-For', filename) subject = subject.strip() to_field = list(to_field) cc_field = list(cc_field) if settings.DEFAULT_FROM_EMAIL: sender = build_email_address(full_name=user.get_full_name(), email=settings.DEFAULT_FROM_EMAIL) else: sender = None message = EmailMessage(subject=subject, text_body=text_body.encode('utf-8'), html_body=html_body.encode('utf-8'), from_email=from_email, sender=sender, to=to_field, cc=cc_field, in_reply_to=in_reply_to, headers=headers) try: message.send() except Exception: logging.exception( "Error sending e-mail notification with subject " "'%s' on behalf of '%s' to '%s'", subject, from_email, ','.join(to_field + cc_field)) return message.message_id
class MultiPartParser(object): """ A rfc2388 multipart/form-data parser. ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ def __init__(self, META, input_data, upload_handlers, encoding=None): """ Initialize the MultiPartParser object. :META: The standard ``META`` dictionary in Django request objects. :input_data: The raw post data, as a file-like object. :upload_handlers: A list of UploadHandler instances that perform operations on the uploaded data. :encoding: The encoding with which to treat the incoming data. """ # # Content-Type should contain multipart and the boundary information. # content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', '')) if not content_type.startswith('multipart/'): raise MultiPartParserError('Invalid Content-Type: %s' % content_type) # Parse the header to get the boundary to split the parts. ctypes, opts = parse_header(content_type.encode('ascii')) boundary = opts.get('boundary') if not boundary or not cgi.valid_boundary(boundary): raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary) # Content-Length should contain the length of the body we are about # to receive. try: content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH', 0))) except (ValueError, TypeError): content_length = 0 if content_length < 0: # This means we shouldn't continue...raise an error. raise MultiPartParserError("Invalid content length: %r" % content_length) if isinstance(boundary, six.text_type): boundary = boundary.encode('ascii') self._boundary = boundary self._input_data = input_data # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] self._chunk_size = min([2 ** 31 - 4] + possible_sizes) self._meta = META self._encoding = encoding or settings.DEFAULT_CHARSET self._content_length = content_length self._upload_handlers = upload_handlers def parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. Returns a tuple containing the POST and FILES dictionary, respectively. """ # We have to import QueryDict down here to avoid a circular import. from django.http import QueryDict encoding = self._encoding handlers = self._upload_handlers # HTTP spec says that Content-Length >= 0 is valid # handling content-length == 0 before continuing if self._content_length == 0: return QueryDict('', encoding=self._encoding), MultiValueDict() # See if any of the handlers take care of the parsing. # This allows overriding everything if need be. for handler in handlers: result = handler.handle_raw_input(self._input_data, self._meta, self._content_length, self._boundary, encoding) # Check to see if it was handled if result is not None: return result[0], result[1] # Create the data structures to be used later. self._post = QueryDict('', mutable=True) self._files = MultiValueDict() # Instantiate the parser and stream: stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None counters = [0] * len(handlers) try: for item_type, meta_data, field_stream in Parser(stream, self._boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. self.handle_file_complete(old_field_name, counters) old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # This is a post field, we can just set it in the post if transfer_encoding == 'base64': raw_data = field_stream.read() try: data = base64.b64decode(raw_data) except _BASE64_DECODE_ERROR: data = raw_data else: data = field_stream.read() self._post.appendlist(field_name, force_text(data, encoding, errors='replace')) elif item_type == FILE: # This is a file, use the handler... file_name = disposition.get('filename') if not file_name: continue file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type, content_type_extra = meta_data.get('content-type', ('', {})) content_type = content_type.strip() charset = content_type_extra.get('charset') try: content_length = int(meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): content_length = None counters = [0] * len(handlers) try: for handler in handlers: try: handler.new_file(field_name, file_name, content_type, content_length, charset, content_type_extra) except StopFutureHandlers: break for chunk in field_stream: if transfer_encoding == 'base64': # We only special-case base64 transfer encoding # We should always decode base64 chunks by multiple of 4, # ignoring whitespace. stripped_chunk = b"".join(chunk.split()) remaining = len(stripped_chunk) % 4 while remaining != 0: over_chunk = field_stream.read(4 - remaining) stripped_chunk += b"".join(over_chunk.split()) remaining = len(stripped_chunk) % 4 try: chunk = base64.b64decode(stripped_chunk) except Exception as e: # Since this is only a chunk, any error is an unfixable error. msg = "Could not decode base64 data: %r" % e six.reraise(MultiPartParserError, MultiPartParserError(msg), sys.exc_info()[2]) for i, handler in enumerate(handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: # If the chunk received by the handler is None, then don't continue. break except SkipFile: self._close_files() # Just use up the rest of this file... exhaust(field_stream) else: # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) except StopUpload as e: self._close_files() if not e.connection_reset: exhaust(self._input_data) else: # Make sure that the request data is all fed exhaust(self._input_data) # Signal that the upload has completed. for handler in handlers: retval = handler.upload_complete() if retval: break return self._post, self._files def handle_file_complete(self, old_field_name, counters): """ Handle all the signaling that takes place when a file is complete. """ for i, handler in enumerate(self._upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: # If it returns a file object, then set the files dict. self._files.appendlist( force_text(old_field_name, self._encoding, errors='replace'), file_obj) break def IE_sanitize(self, filename): """Cleanup filename from Internet Explorer full paths.""" return filename and filename[filename.rfind("\\") + 1:].strip() def _close_files(self): # Free up all file handles. # FIXME: this currently assumes that upload handlers store the file as 'file' # We should document that... (Maybe add handler.free_file to complement new_file) for handler in self._upload_handlers: if hasattr(handler, 'file'): handler.file.close()
def resolve_params(self, context): " Parse the parameters into a dict. " params = MultiValueDict() for key, value in self.parse_params(self.nodelist.render(context)): params.appendlist(key, value) return params
class MultiPartParser: """ A rfc2388 multipart/form-data parser. ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ def __init__(self, META, input_data, upload_handlers, encoding=None): """ Initialize the MultiPartParser object. :META: The standard ``META`` dictionary in Django request objects. :input_data: The raw post data, as a file-like object. :upload_handlers: A list of UploadHandler instances that perform operations on the uploaded data. :encoding: The encoding with which to treat the incoming data. """ # Content-Type should contain multipart and the boundary information. content_type = META.get('CONTENT_TYPE', '') if not content_type.startswith('multipart/'): raise MultiPartParserError('Invalid Content-Type: %s' % content_type) # Parse the header to get the boundary to split the parts. try: ctypes, opts = parse_header(content_type.encode('ascii')) except UnicodeEncodeError: raise MultiPartParserError( 'Invalid non-ASCII Content-Type in multipart: %s' % force_str(content_type)) boundary = opts.get('boundary') if not boundary or not cgi.valid_boundary(boundary): raise MultiPartParserError('Invalid boundary in multipart: %s' % force_str(boundary)) # Content-Length should contain the length of the body we are about # to receive. try: content_length = int(META.get('CONTENT_LENGTH', 0)) except (ValueError, TypeError): content_length = 0 if content_length < 0: # This means we shouldn't continue...raise an error. raise MultiPartParserError("Invalid content length: %r" % content_length) if isinstance(boundary, str): boundary = boundary.encode('ascii') self._boundary = boundary self._input_data = input_data # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [ x.chunk_size for x in upload_handlers if x.chunk_size ] self._chunk_size = min([2**31 - 4] + possible_sizes) self._meta = META self._encoding = encoding or settings.DEFAULT_CHARSET self._content_length = content_length self._upload_handlers = upload_handlers def parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. Return a tuple containing the POST and FILES dictionary, respectively. """ from django.http import QueryDict encoding = self._encoding handlers = self._upload_handlers # HTTP spec says that Content-Length >= 0 is valid # handling content-length == 0 before continuing if self._content_length == 0: return QueryDict(encoding=self._encoding), MultiValueDict() # See if any of the handlers take care of the parsing. # This allows overriding everything if need be. for handler in handlers: result = handler.handle_raw_input( self._input_data, self._meta, self._content_length, self._boundary, encoding, ) # Check to see if it was handled if result is not None: return result[0], result[1] # Create the data structures to be used later. self._post = QueryDict(mutable=True) self._files = MultiValueDict() # Instantiate the parser and stream: stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None counters = [0] * len(handlers) # Number of bytes that have been read. num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 # To limit the amount of data read from the request. read_size = None try: for item_type, meta_data, field_stream in Parser( stream, self._boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. self.handle_file_complete(old_field_name, counters) old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_str(field_name, encoding, errors='replace') if item_type == FIELD: # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. num_post_keys += 1 if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): raise TooManyFieldsSent( 'The number of GET/POST parameters exceeded ' 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.') # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read # This is a post field, we can just set it in the post if transfer_encoding == 'base64': raw_data = field_stream.read(size=read_size) num_bytes_read += len(raw_data) try: data = base64.b64decode(raw_data) except binascii.Error: data = raw_data else: data = field_stream.read(size=read_size) num_bytes_read += len(data) # Add two here to make the check consistent with the # x-www-form-urlencoded check that includes '&='. num_bytes_read += len(field_name) + 2 if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE): raise RequestDataTooBig( 'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.' ) self._post.appendlist( field_name, force_str(data, encoding, errors='replace')) elif item_type == FILE: # This is a file, use the handler... file_name = disposition.get('filename') if file_name: file_name = force_str(file_name, encoding, errors='replace') file_name = self.IE_sanitize(html.unescape(file_name)) if not file_name: continue content_type, content_type_extra = meta_data.get( 'content-type', ('', {})) content_type = content_type.strip() charset = content_type_extra.get('charset') try: content_length = int( meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): content_length = None counters = [0] * len(handlers) try: for handler in handlers: try: handler.new_file( field_name, file_name, content_type, content_length, charset, content_type_extra, ) except StopFutureHandlers: break for chunk in field_stream: if transfer_encoding == 'base64': # We only special-case base64 transfer encoding # We should always decode base64 chunks by multiple of 4, # ignoring whitespace. stripped_chunk = b"".join(chunk.split()) remaining = len(stripped_chunk) % 4 while remaining != 0: over_chunk = field_stream.read(4 - remaining) stripped_chunk += b"".join( over_chunk.split()) remaining = len(stripped_chunk) % 4 try: chunk = base64.b64decode(stripped_chunk) except Exception as exc: # Since this is only a chunk, any error is an unfixable error. raise MultiPartParserError( "Could not decode base64 data." ) from exc for i, handler in enumerate(handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk( chunk, counters[i]) counters[i] += chunk_length if chunk is None: # Don't continue if the chunk received by # the handler is None. break except SkipFile: self._close_files() # Just use up the rest of this file... exhaust(field_stream) else: # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) except StopUpload as e: self._close_files() if not e.connection_reset: exhaust(self._input_data) else: # Make sure that the request data is all fed exhaust(self._input_data) # Signal that the upload has completed. # any() shortcircuits if a handler's upload_complete() returns a value. any(handler.upload_complete() for handler in handlers) self._post._mutable = False return self._post, self._files def handle_file_complete(self, old_field_name, counters): """ Handle all the signaling that takes place when a file is complete. """ for i, handler in enumerate(self._upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: # If it returns a file object, then set the files dict. self._files.appendlist( force_str(old_field_name, self._encoding, errors='replace'), file_obj) break def IE_sanitize(self, filename): """Cleanup filename from Internet Explorer full paths.""" return filename and filename[filename.rfind("\\") + 1:].strip() def _close_files(self): # Free up all file handles. # FIXME: this currently assumes that upload handlers store the file as 'file' # We should document that... (Maybe add handler.free_file to complement new_file) for handler in self._upload_handlers: if hasattr(handler, 'file'): handler.file.close()
class IntelligentUploadHandler(FileUploadHandler): """ An upload handler which overrides the default multipart parser to allow simultaneous parsing of fields and files... intelligently. Subclass this for real and true awesomeness. """ def __init__(self, *args, **kwargs): super(IntelligentUploadHandler, self).__init__(*args, **kwargs) def field_parsed(self, field_name, field_value): """ A callback method triggered when a non-file field has been parsed successfully by the parser. Use this to listen for new fields being parsed. """ pass def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): """ Parse the raw input from the HTTP request and split items into fields and files, executing callback methods as necessary. Shamelessly adapted and borrowed from django.http.multiparser.MultiPartParser. """ # following suit from the source class, this is imported here to avoid # a potential circular import from django.http import QueryDict # create return values self.POST = QueryDict('', mutable=True) self.FILES = MultiValueDict() # initialize the parser and stream stream = LazyStream(ChunkIter(input_data, self.chunk_size)) # whether or not to signal a file-completion at the beginning # of the loop. old_field_name = None counter = 0 try: for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # we run this test at the beginning of the next loop since # we cannot be sure a file is complete until we hit the # next boundary/part of the multipart content. file_obj = self.file_complete(counter) if file_obj: # if we return a file object, add it to the files dict self.FILES.appendlist( force_text(old_field_name, encoding, errors='replace'), file_obj) # wipe it out to prevent havoc old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # this is a POST field if transfer_encoding == "base64": raw_data = field_stream.read() try: data = str(raw_data).decode('base64') except: data = raw_data else: data = field_stream.read() self.POST.appendlist( field_name, force_text(data, encoding, errors='replace')) # trigger listener self.field_parsed(field_name, self.POST.get(field_name)) elif item_type == FILE: # this is a file file_name = disposition.get('filename') if not file_name: continue # transform the file name file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type = meta_data.get('content-type', ('', ))[0].strip() try: charset = meta_data.get('content-type', (0, {}))[1]\ .get('charset', None) except: charset = None try: file_content_length = int( meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): file_content_length = None counter = 0 # now, do the important file stuff try: # alert on the new file kwargs = { 'content_type': content_type, 'content_length': file_content_length, 'charset': charset } self.new_file(field_name, file_name, **kwargs) # chubber-chunk it for chunk in field_stream: # we need AES compatibles blocks (multiples of 16 bits) over_bytes = len(chunk) % 16 if over_bytes: over_chunk =\ field_stream.read(16 - over_bytes) chunk += over_chunk if transfer_encoding == "base64": try: chunk = base64.b64decode(chunk) except Exception as e: # since this is anly a chunk, any # error is an unfixable error raise MultiPartParserError( "Could not decode base64 data: %r" % e) chunk_length = len(chunk) self.receive_data_chunk(chunk, counter) counter += chunk_length if counter > settings.UPLOAD_FILE_SIZE_LIMIT: raise SkipFile('File is too big.') # ... and we're done except SkipFile: # just eat the rest exhaust(field_stream) else: # handle file upload completions on next iteration old_field_name = field_name except StopUpload as e: # if we get a request to stop the upload, # exhaust it if no con reset if not e.connection_reset: exhaust(input_data) else: # make sure that the request data is all fed exhaust(input_data) # signal the upload has been completed self.upload_complete() return self.POST, self.FILES def IE_sanitize(self, filename): """Cleanup filename from Internet Explorer full paths.""" return filename and filename[filename.rfind("\\") + 1:].strip()
class EmailBaseModel(models.Model): sender = models.EmailField(_("sender"), max_length=255) recipient = models.CharField(_("recipient"), max_length=255) subject = models.CharField(_("subject"), max_length=255, blank=True) from_str = models.CharField(_("from"), max_length=255) body_plain = models.TextField(_("body plain"), blank=True) body_html = models.TextField(_("body html"), blank=True) stripped_text = models.TextField(_("stripped text"), blank=True) stripped_html = models.TextField(_("stripped html"), blank=True) stripped_signature = models.TextField(_("stripped signature"), blank=True) message_headers = models.TextField(_("message headers"), blank=True, help_text=_("Stored in JSON.")) content_id_map = models.TextField( _("Content-ID map"), blank=True, help_text= _("Dictionary mapping Content-ID (CID) values to corresponding attachments. Stored in JSON." )) received = models.DateTimeField(_("received"), auto_now_add=True) class Meta: abstract = True verbose_name = _("incoming email") verbose_name_plural = _("incoming emails") def __init__(self, *args, **kwargs): super(EmailBaseModel, self).__init__(*args, **kwargs) self._headers = None self._cids = None def _load_headers(self): self._headers = MultiValueDict() try: header_list = json.loads(self.message_headers) for key, val in header_list: self._headers.appendlist(key, val) except: logger.exception( "Error parsing JSON data containing message headers") @property def headers(self): """Access message_headers parsed into MultiValueDict""" if self._headers is None: self._load_headers() return self._headers def _load_cids(self): if self.content_id_map: self._cids = {} try: self._cids = json.loads(self.content_id_map) except: logger.exception("Error parsing JSON data containing Content-IDs") @property def content_ids(self): """Access content_id_map as dict""" if not self.content_id_map: return if self._cids is None: self._load_cids() return self._cids def __unicode__(self): return _("Message from {from_str}: {subject_trunc}").format( from_str=self.from_str, subject_trunc=self.subject[:20])
class EmailMessage(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True, verbose_name=_("user"), on_delete=models.SET_NULL) sender = models.EmailField(_("sender"), max_length=255) from_str = models.CharField(_("from"), max_length=255) recipient = models.CharField(_("recipient"), max_length=255) subject = models.CharField(_("subject"), max_length=255, blank=True) body_plain = models.TextField(_("body plain"), blank=True) body_html = models.TextField(_("body html"), blank=True) stripped_text = models.TextField(_("stripped text"), blank=True) stripped_html = models.TextField(_("stripped html"), blank=True) stripped_signature = models.TextField(_("stripped signature"), blank=True) message_headers = models.TextField(_("message headers"), blank=True, help_text=_("Stored in JSON.")) content_id_map = models.TextField( _("Content-ID map"), blank=True, help_text= _("Dictionary mapping Content-ID (CID) values to corresponding attachments. Stored in JSON." )) received = models.DateTimeField(_("received"), auto_now_add=True) mailing_list = models.ForeignKey(MailingList, blank=True, null=True, on_delete=models.CASCADE) class Meta: verbose_name = _("email message") verbose_name_plural = _("email messages") def __init__(self, *args, **kwargs): super(EmailMessage, self).__init__(*args, **kwargs) self._headers = None self._cids = None def _load_headers(self): self._headers = MultiValueDict() try: header_list = json.loads(self.message_headers) for key, val in header_list: self._headers.appendlist(key, val) except: logger.exception( "Error parsing JSON data containing message headers") def _load_cids(self): if self.content_id_map: self._cids = {} try: self._cids = json.loads(self.content_id_map) except: logger.exception("Error parsing JSON data containing Content-IDs") @property def headers(self): """Access message_headers parsed into MultiValueDict""" if self._headers is None: self._load_headers() return self._headers @property def content_ids(self): """Access content_id_map as dict""" if not self.content_id_map: return if self._cids is None: self._load_cids() return self._cids @property def message_id(self): return self.headers.get('Message-Id', None) @property def cc(self): return self.headers.get('Cc', None) @property def references(self): return self.headers.get('References', None) @property def in_reply_to(self): return self.headers.get('In-Reply-To', None) @property def from_name(self): from_name, from_address = email.utils.parseaddr(self.from_str) return from_name @property def from_address(self): from_name, from_address = email.utils.parseaddr(self.from_str) return from_address @property def clean_subject(self): subject = self.subject if self.mailing_list and self.mailing_list.subject_prefix: prefix = self.mailing_list.subject_prefix index = subject.find(prefix) if index >= 0: subject = subject[index + len(prefix):] return subject.strip() @property def is_moderated_subject(self): s = self.subject.lower() if "auto-reply" in s: return True if "auto reply" in s: return True if "automatic reply" in s: return True if "out of office" in s: return True return False @property def public_url(self): return settings.SITE_PROTO + "://" + settings.SITE_DOMAIN + reverse( 'comlink:mail', kwargs={'id': self.id}) def get_user(self): if not self.user: self.user = User.helper.by_email(self.from_address) return self.user def get_body(self, prefer_html=True): if prefer_html: if self.stripped_html: return self.stripped_html if self.body_html: return self.body_html if self.stripped_text: return self.stripped_text return self.body_plain def get_mailgun_data(self, stripped=True): if stripped: body_plain = self.stripped_text body_html = self.stripped_html else: body_plain = self.body_plain body_html = self.body_html # Build and return our data mailgun_data = { "from": self.from_str, "to": [ self.recipient, ], "cc": [ self.cc, ], "subject": self.clean_subject, "text": body_plain, "html": body_html, } return mailgun_data def __str__(self): return _("Message from {from_str}: {subject_trunc}").format( from_str=self.from_str, subject_trunc=self.subject[:20])