def view_index(request): if not utils.docutils_is_available: return missing_docutils_page(request) view_dict = {} view_functions = extract_views_from_urlpatterns(api1_urls.urlpatterns) + extract_views_from_urlpatterns( oauth_urls.urlpatterns ) for (func, regex) in view_functions: view = { "name": getattr(func, "__name__", func.__class__.__name__), "module": func.__module__, "url": simplify_regex(regex), } if view["name"] in ["authorize", "access_token", "request_token"]: view["url"] = "/oauth%s" % view["url"] view["group"] = "oauth" else: view["group"] = view["url"].split("/")[1] view["url"] = "/1" + view["url"] if view_dict.has_key(view["group"]): view_dict[view["group"]].append(view) else: view_dict[view["group"]] = [view] return render_to_response( "doc/view_index.tpl", {"root_path": get_root_path(), "view_dict": view_dict}, context_instance=RequestContext(request), )
def view_index(request): if not utils.docutils_is_available: return missing_docutils_page(request) view_dict = {} view_functions = extract_views_from_urlpatterns(api1_urls.urlpatterns) + \ extract_views_from_urlpatterns(oauth_urls.urlpatterns) for (func, regex) in view_functions: view = { 'name': getattr(func, '__name__', func.__class__.__name__), 'module': func.__module__, 'url': simplify_regex(regex) } if view['name'] in ['authorize', 'access_token', 'request_token']: view['url'] = "/oauth%s" % view['url'] view['group'] = 'oauth' else: view['group'] = view['url'].split('/')[1] view['url'] = "/1" + view['url'] if view_dict.has_key(view['group']): view_dict[view['group']].append(view) else: view_dict[view['group']] = [view] return render_to_response( 'doc/view_index.tpl', { 'root_path': get_root_path(), 'view_dict': view_dict }, context_instance=RequestContext(request) )
def handle(self, *args, **options): verbosity = int(options.get('verbosity', 1)) if len(args) > 1: raise CommandError('Only one start url is currently supported.') else: start_url = args[0] if args else '/' if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] conf_urls = {} for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e))) continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: #Get function name and add it to the hash of URLConf urls func_name = hasattr(func, '__name__') and func.__name__ or repr(func) conf_urls[regex] = ['func.__module__', func_name]
def handle(self, *args, **options): if args: appname, = args style = color_style() if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] views = [] for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: if options.get('traceback', None): import traceback self.stderr.write(traceback.format_exc() + "\n") self.stderr.write( style.ERROR("Error occurred while trying to load %s: %s\n" % (settings_mod.ROOT_URLCONF, str(e)))) continue view_functions = \ extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: func_name = hasattr(func, '__name__') and \ func.__name__ or repr(func) views.append("%(url)s\t%(module)s.%(name)s" % {'name': style.MODULE_NAME(func_name), 'module': style.MODULE( getattr(func, '__module__', '<no module>')), 'url': style.URL(simplify_regex(regex))})
def handle(self, *args, **options): verbosity = int(options.get("verbosity", 1)) if len(args) > 1: raise CommandError("Only one start url is currently supported.") else: start_url = args[0] if args else "/" if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, [""]) for m in settings.ADMIN_FOR] else: settings_modules = [settings] conf_urls = {} for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [""]) except Exception, e: print("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e))) continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: # Get function name and add it to the hash of URLConf urls func_name = hasattr(func, "__name__") and func.__name__ or repr(func) conf_urls[regex] = ["func.__module__", func_name]
def handle(self, *args, **options): USE_PDB = options.get("pdb", False) MAKE_FIXTURES = options.get("fixtures", False) CHECK_HTML = options.get("html", False) CHECK_TIME = options.get("time", False) STORE_RESPONSE = options.get("response", False) VERBOSITY = int(options.get("verbosity", 1)) # EACH_URL = options.get('each', 100000) if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, [""]) for m in settings.ADMIN_FOR] else: settings_modules = [settings] conf_urls = {} for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [""]) except Exception, e: print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e))) continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: # Get function name and add it to the hash of URLConf urls func_name = hasattr(func, "__name__") and func.__name__ or repr(func) conf_urls[regex] = ["func.__module__", func_name]
def _test_url(self, urlpatterns): deconstructed_urls = extract_views_from_urlpatterns(urlpatterns) added_kwargs = {} # we need to format urls with proper ids url_ctx = self.get_url_path_ctx() if url_ctx: self.response_data = { url.format(**url_ctx): data for url, data in self.response_data.items() } for (view, regex, namespace, name) in deconstructed_urls: request_data = self.request_data.get(name, {}).copy() for key in list(re.compile(regex).groupindex.keys()): if key in list(request_data.keys()): added_kwargs[key] = request_data[key] continue if key not in self.default_kwargs: raise Exception( 'URL argument not in test kwargs. Please add `%s`' % key) added_kwargs[key] = self.default_kwargs[key] path = reverse(name, kwargs=added_kwargs) self.assertResponse(path=path, name=name) added_kwargs = {}
def handle(self, *args, **options): if args: appname, = args style = color_style() if settings.ADMIN_FOR: settings_modules = [ __import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR ] else: settings_modules = [settings] views = [] for settings_mod in settings_modules: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) view_functions = extract_views_from_urlpatterns( urlconf.urlpatterns) for (func, regex) in view_functions: views.append( "%(url)s\t%(module)s.%(name)s" % { 'name': style.MODULE_NAME(func.__name__), 'module': style.MODULE(func.__module__), 'url': style.URL(simplify_regex(regex)) }) return "\n".join([v for v in views])
def handle(self, *args, **options): verbosity = int(options.get('verbosity', 1)) depth = int(options.get('depth', 3)) auth = _parse_auth(options.get('auth')) if verbosity > 1: log_level = logging.DEBUG elif verbosity: log_level = logging.INFO else: log_level = logging.WARN crawl_logger = logging.getLogger('crawler') crawl_logger.setLevel(logging.DEBUG) crawl_logger.propagate = 0 log_stats = LogStatsHandler() crawl_logger.addHandler(log_stats) console = logging.StreamHandler() console.setLevel(log_level) console.setFormatter( logging.Formatter( "%(name)s [%(levelname)s] %(module)s: %(message)s")) crawl_logger.addHandler(console) if len(args) > 1: raise CommandError('Only one start url is currently supported.') else: start_url = args[0] if args else '/' if getattr(settings, 'ADMIN_FOR', None): settings_modules = [ __import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR ] else: settings_modules = [settings] conf_urls = {} # Build the list URLs to test from urlpatterns: for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: logging.exception("Error occurred while trying to load %s: %s", settings_mod.ROOT_URLCONF, str(e)) continue view_functions = extract_views_from_urlpatterns( urlconf.urlpatterns) for (func, regex, namespace, name) in view_functions: #Get function name and add it to the hash of URLConf urls func_name = hasattr(func, '__name__') and func.__name__ or repr(func) conf_urls[regex] = ['func.__module__', func_name]
def get_url_name_choices(self): urlconf = import_module(settings.ROOT_URLCONF) ignored = self.Meta.model.URL_NAME_IGNORED or () choices = [(':'.join(ns + [name]), path) for func, path, ns, name in extract_views_from_urlpatterns(urlconf.urlpatterns) if name and not ('(?P' in path or '<' in path)] return [(name, '/%s (%s)' % (url, name)) for name, url in choices if not ignored or not any(re.match(i, name) for i in ignored)]
def handle(self, *args, **options): verbosity = int(options.get('verbosity', 1)) depth = int(options.get('depth', 3)) auth = _parse_auth(options.get('auth')) if verbosity == 3: log_level = 1 elif verbosity == 2: log_level = logging.DEBUG elif verbosity: log_level = logging.INFO else: log_level = logging.WARN crawl_logger = logging.getLogger('crawler') crawl_logger.setLevel(logging.DEBUG) crawl_logger.propagate = 0 log_stats = LogStatsHandler() crawl_logger.addHandler(log_stats) console = logging.StreamHandler() console.setLevel(log_level) console.setFormatter(logging.Formatter("%(name)s [%(levelname)s] %(module)s: %(message)s")) crawl_logger.addHandler(console) if len(args) > 1: raise CommandError('Only one start url is currently supported.') else: start_url = args[0] if args else '/' if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] conf_urls = {} # Build the list URLs to test from urlpatterns: for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: logging.exception("Error occurred while trying to load %s: %s", settings_mod.ROOT_URLCONF, str(e)) continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: #Get function name and add it to the hash of URLConf urls func_name = hasattr(func, '__name__') and func.__name__ or repr(func) conf_urls[regex] = ['func.__module__', func_name]
def get_aio_routes(patterns=None): """Walk the URL patterns to find any coroutine views.""" if patterns is None: urlconf = import_module(settings.ROOT_URLCONF) patterns = urlconf.urlpatterns routes = [] view_functions = extract_views_from_urlpatterns(patterns) for (func, regex, namespace, name) in view_functions: if asyncio.iscoroutinefunction(func) or inspect.isgeneratorfunction(func): routes.append(DjangoRegexRoute('*', func, name, regex)) return routes
def get_aio_routes(patterns=None): """Walk the URL patterns to find any coroutine views.""" if patterns is None: urlconf = import_module(settings.ROOT_URLCONF) patterns = urlconf.urlpatterns routes = [] view_functions = extract_views_from_urlpatterns(patterns) for (func, regex, namespace, name) in view_functions: if asyncio.iscoroutinefunction(func) or inspect.isgeneratorfunction( func): routes.append(DjangoRegexRoute('*', func, name, regex)) return routes
def check_policy_for_all_routes(): errors = [] urlconf = import_module(settings.ROOT_URLCONF) # extract_views_from_urlpatterns is an undocumented internal from the # contrib 'admindocs' module that happens to be useful for demo purposes. for (func, regex, namespace, name) in extract_views_from_urlpatterns(urlconf.urlpatterns): if not regex.startswith('policies/'): continue if not has_security_policy_applied(func): errors.append(( f"{func.__module__}.{func.__name__} needs to have a security policy applied", regex), ) return errors
def _test_url(self, urlpatterns): deconstructed_urls = extract_views_from_urlpatterns(urlpatterns) added_kwargs = {} for (view, regex, namespace, name) in deconstructed_urls: request_data = self.request_data.get(name, {}).copy() for key in list(re.compile(regex).groupindex.keys()): if key in list(request_data.keys()): added_kwargs[key] = request_data[key] continue if key not in self.default_kwargs: raise Exception('URL argument not in test kwargs. Please add `%s`' % key) added_kwargs[key] = self.default_kwargs[key] path = reverse(name, kwargs=added_kwargs) self.assertResponse(path=path, name=name) added_kwargs = {}
def get_service_endpoint_urls(urlconfig=None, default_value='1'): """ This function finds all endpoint urls in a service. Args: urlconfig: A django url config module to use. Defaults to settings.ROOT_URLCONF default_value: A string value to replace all url parameters with Returns: A list of urls with path parameters (i.e. <pk>) replaced with default_value """ if not urlconfig: urlconfig = getattr(settings, 'ROOT_URLCONF') try: urlconfig_mod = import_module(urlconfig) except Exception as ex: raise Exception( "Unable to import url config module. Url Config: {0}. Message: {1}" .format(urlconfig, ex.message)) extracted_views = extract_views_from_urlpatterns(urlconfig_mod.urlpatterns) views_regex_url_patterns = [item[1] for item in extracted_views] simplified_regex_url_patterns = [ simplify_regex(pattern) for pattern in views_regex_url_patterns ] # Strip out urls we don't need to test. result_urls = [] pattern = re.compile(r'<\w+>') for url in simplified_regex_url_patterns: if url.find('<format>') != -1: continue if url == u'/': continue if url == u'/health': continue parameters = pattern.findall(url) for param in parameters: url = url.replace(param, default_value) result_urls.append(url) return result_urls
def _get_urls(self): urls = [] if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception as e: continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for view_function in view_functions: urls.append(simplify_regex(view_function[1])) return urls
def get_service_endpoint_urls(urlconfig=None, default_value='1'): """ This function finds all endpoint urls in a service. Args: urlconfig: A django url config module to use. Defaults to settings.ROOT_URLCONF default_value: A string value to replace all url parameters with Returns: A list of urls with path parameters (i.e. <pk>) replaced with default_value """ if not urlconfig: urlconfig = getattr(settings, 'ROOT_URLCONF') try: urlconfig_mod = import_module(urlconfig) except Exception as ex: raise Exception( "Unable to import url config module. Url Config: {0}. Message: {1}".format(urlconfig, ex.message)) extracted_views = extract_views_from_urlpatterns(urlconfig_mod.urlpatterns) views_regex_url_patterns = [item[1] for item in extracted_views] simplified_regex_url_patterns = [simplify_regex(pattern) for pattern in views_regex_url_patterns] # Strip out urls we don't need to test. result_urls = [] pattern = re.compile(r'<\w+>') for url in simplified_regex_url_patterns: if url.find('<format>') != -1: continue if url == u'/': continue if url == u'/health': continue parameters = pattern.findall(url) for param in parameters: url = url.replace(param, default_value) result_urls.append(url) return result_urls
def handle(self, *args, **options): if args: appname, = args style = color_style() if settings.ADMIN_FOR: settings_modules = [ __import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR ] else: settings_modules = [settings] views = [] for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: if options.get('traceback', None): import traceback self.stderr.write(traceback.format_exc() + "\n") self.stderr.write( style.ERROR( "Error occurred while trying to load %s: %s\n" % (settings_mod.ROOT_URLCONF, str(e)))) continue view_functions = \ extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: func_name = hasattr(func, '__name__') and \ func.__name__ or repr(func) views.append( "%(url)s\t%(module)s.%(name)s" % { 'name': style.MODULE_NAME(func_name), 'module': style.MODULE(getattr(func, '__module__', '<no module>')), 'url': style.URL(simplify_regex(regex)) })
def handle(self, *args, **options): if args: appname, = args style = color_style() if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] views = [] for settings_mod in settings_modules: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: views.append("%(url)s\t%(module)s.%(name)s" % {'name': style.MODULE_NAME(func.__name__), 'module': style.MODULE(func.__module__), 'url': style.URL(simplify_regex(regex))}) return "\n".join([v for v in views])
def get_urls(urlconf_name): urlconf = import_module(urlconf_name) all_urls = extract_views_from_urlpatterns(urlconf.urlpatterns) for mounted_url in all_urls: viewfunc = mounted_url[0] regex = mounted_url[1] try: namespace = mounted_url[2] except IndexError: namespace = None if namespace is None: namespace = [] try: name = mounted_url[3] except IndexError: name = None modname = viewfunc.__module__ viewname = getattr(viewfunc, '__name__', viewfunc.__class__.__name__) simple_url = simplify_regex(regex) named_groups = all_params_re.findall(simple_url) all_groups = (replace_simple_regex(var, index) for index, var in enumerate(named_groups, start=1)) textsearch = set() textsearch.update(modname.lower().split('.')) textsearch.add(viewname.lower()) if name: textsearch.add(name.lower()) textsearch.update(x.lower() for x in simple_url.split('/') if "<" not in x and x.strip() != '') yield { 'textsearch': " ".join(textsearch), 'textdata': textsearch, 'module': modname, 'view': viewname, 'url': simple_url, 'namespace': namespace, 'namespace_string': ":".join(namespace), 'name': name, 'named_url_kwargs': tuple(all_groups), }
def handle(self, *args, **options): verbosity = int(options.get('verbosity', 1)) if settings.ADMIN_FOR: settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] else: settings_modules = [settings] conf_urls = {} for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception, e: print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e))) continue view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) for (func, regex) in view_functions: #Get function name and add it to the hash of URLConf urls func_name = hasattr(func, '__name__') and func.__name__ or repr(func) conf_urls[regex] = ['func.__module__', func_name]
def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None): # taken from 16a842b3795ca78a5918538ab6b9f1afbd718f72 views = [] for p in urlpatterns: if hasattr(p, 'url_patterns'): try: patterns = p.url_patterns except ImportError: continue views.extend( extract_views_from_urlpatterns( patterns, base + p.regex.pattern, (namespace or []) + (p.namespace and [p.namespace] or []))) elif hasattr(p, 'callback'): try: views.append((p.callback, base + p.regex.pattern, namespace, p.name)) except ViewDoesNotExist: continue else: raise TypeError( _("%s does not appear to be a urlpattern object") % p) return views
def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None): # taken from 16a842b3795ca78a5918538ab6b9f1afbd718f72 views = [] for p in urlpatterns: if hasattr(p, 'url_patterns'): try: patterns = p.url_patterns except ImportError: continue views.extend(extract_views_from_urlpatterns( patterns, base + p.regex.pattern, (namespace or []) + (p.namespace and [p.namespace] or []) )) elif hasattr(p, 'callback'): try: views.append((p.callback, base + p.regex.pattern, namespace, p.name)) except ViewDoesNotExist: continue else: raise TypeError(_("%s does not appear to be a urlpattern object") % p) return views
def _test_url(self, urlpatterns): deconstructed_urls = extract_views_from_urlpatterns(urlpatterns) added_kwargs = {} # we need to format urls with proper ids url_ctx = self.get_url_path_ctx() if url_ctx: self.response_data = { url.format(**url_ctx): data for url, data in self.response_data.items() } for (view, regex, namespace, name) in deconstructed_urls: request_data = self.request_data.get(name, {}).copy() for key in list(re.compile(regex).groupindex.keys()): if key in list(request_data.keys()): added_kwargs[key] = request_data[key] continue if key not in self.default_kwargs: raise Exception('URL argument not in test kwargs. Please add `%s`' % key) added_kwargs[key] = self.default_kwargs[key] path = reverse(name, kwargs=added_kwargs) self.assertResponse(path=path, name=name) added_kwargs = {}
def handle(self, *args, **options): verbosity = int(options.get('verbosity', 1)) depth = int(options.get('depth', 3)) auth = _parse_auth(options.get('auth')) if verbosity > 1: log_level = logging.DEBUG elif verbosity: log_level = logging.INFO else: log_level = logging.WARN crawl_logger = logging.getLogger('crawler') crawl_logger.setLevel(logging.DEBUG) crawl_logger.propagate = 0 log_stats = LogStatsHandler() crawl_logger.addHandler(log_stats) console = logging.StreamHandler() console.setLevel(log_level) console.setFormatter( logging.Formatter( "%(name)s [%(levelname)s] %(module)s: %(message)s")) crawl_logger.addHandler(console) if len(args) > 1: raise CommandError('Only one start url is currently supported.') else: start_url = args[0] if args else '/' if getattr(settings, 'ADMIN_FOR', None): settings_modules = [ __import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR ] else: settings_modules = [settings] conf_urls = {} # Build the list URLs to test from urlpatterns: for settings_mod in settings_modules: try: urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) except Exception as e: logging.exception("Error occurred while trying to load %s: %s", settings_mod.ROOT_URLCONF, str(e)) continue view_functions = extract_views_from_urlpatterns( urlconf.urlpatterns) for (func, regex, namespace, name) in view_functions: # Get function name and add it to the hash of URLConf urls func_name = hasattr(func, '__name__') and func.__name__ or repr(func) conf_urls[regex] = ['func.__module__', func_name] c = Crawler( start_url, conf_urls=conf_urls, verbosity=verbosity, output_dir=options.get("output_dir"), ascend=not options.get("no_parent"), auth=auth, ) # Load plugins: for p in options['plugins']: # This nested try is somewhat unsightly but allows easy Pythonic # usage ("--enable-plugin=tidy") instead of Java-esque # "--enable-plugin=test_utils.crawler.plugins.tidy" try: try: plugin_module = __import__(p) except ImportError: if not "." in p: plugin_module = __import__( "test_utils.crawler.plugins.%s" % p, fromlist=["test_utils.crawler.plugins"]) else: raise c.plugins.append(plugin_module.PLUGIN()) except (ImportError, AttributeError) as e: crawl_logger.critical("Unable to load plugin %s: %s", p, e) sys.exit(3) c.run(max_depth=depth) # We'll exit with a non-zero status if we had any errors max_log_level = max(log_stats.stats.keys()) if max_log_level >= logging.ERROR: sys.exit(2) elif max_log_level >= logging.WARNING: sys.exit(1) else: sys.exit(0)