def view_index(request):
if not utils.docutils_is_available:
return missing_docutils_page(request)
view_dict = {}
view_functions = extract_views_from_urlpatterns(api1_urls.urlpatterns) + extract_views_from_urlpatterns(
oauth_urls.urlpatterns
)
for (func, regex) in view_functions:
view = {
"name": getattr(func, "__name__", func.__class__.__name__),
"module": func.__module__,
"url": simplify_regex(regex),
}
if view["name"] in ["authorize", "access_token", "request_token"]:
view["url"] = "/oauth%s" % view["url"]
view["group"] = "oauth"
else:
view["group"] = view["url"].split("/")[1]
view["url"] = "/1" + view["url"]
if view_dict.has_key(view["group"]):
view_dict[view["group"]].append(view)
else:
view_dict[view["group"]] = [view]
return render_to_response(
"doc/view_index.tpl",
{"root_path": get_root_path(), "view_dict": view_dict},
context_instance=RequestContext(request),
)
def view_index(request):
if not utils.docutils_is_available:
return missing_docutils_page(request)
view_dict = {}
view_functions = extract_views_from_urlpatterns(api1_urls.urlpatterns) + \
extract_views_from_urlpatterns(oauth_urls.urlpatterns)
for (func, regex) in view_functions:
view = {
'name': getattr(func, '__name__', func.__class__.__name__),
'module': func.__module__,
'url': simplify_regex(regex)
}
if view['name'] in ['authorize', 'access_token', 'request_token']:
view['url'] = "/oauth%s" % view['url']
view['group'] = 'oauth'
else:
view['group'] = view['url'].split('/')[1]
view['url'] = "/1" + view['url']
if view_dict.has_key(view['group']):
view_dict[view['group']].append(view)
else:
view_dict[view['group']] = [view]
return render_to_response(
'doc/view_index.tpl', {
'root_path': get_root_path(),
'view_dict': view_dict
},
context_instance=RequestContext(request)
)
def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
if len(args) > 1:
raise CommandError('Only one start url is currently supported.')
else:
start_url = args[0] if args else '/'
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
conf_urls = {}
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e)))
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
#Get function name and add it to the hash of URLConf urls
func_name = hasattr(func, '__name__') and func.__name__ or repr(func)
conf_urls[regex] = ['func.__module__', func_name]
def handle(self, *args, **options):
if args:
appname, = args
style = color_style()
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, [''])
for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
views = []
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
if options.get('traceback', None):
import traceback
self.stderr.write(traceback.format_exc() + "\n")
self.stderr.write(
style.ERROR("Error occurred while trying to load %s: %s\n"
% (settings_mod.ROOT_URLCONF, str(e))))
continue
view_functions = \
extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
func_name = hasattr(func, '__name__') and \
func.__name__ or repr(func)
views.append("%(url)s\t%(module)s.%(name)s"
% {'name': style.MODULE_NAME(func_name),
'module': style.MODULE(
getattr(func, '__module__',
'<no module>')),
'url': style.URL(simplify_regex(regex))})
def handle(self, *args, **options):
verbosity = int(options.get("verbosity", 1))
if len(args) > 1:
raise CommandError("Only one start url is currently supported.")
else:
start_url = args[0] if args else "/"
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, [""]) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
conf_urls = {}
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [""])
except Exception, e:
print("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e)))
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
# Get function name and add it to the hash of URLConf urls
func_name = hasattr(func, "__name__") and func.__name__ or repr(func)
conf_urls[regex] = ["func.__module__", func_name]
def handle(self, *args, **options):
USE_PDB = options.get("pdb", False)
MAKE_FIXTURES = options.get("fixtures", False)
CHECK_HTML = options.get("html", False)
CHECK_TIME = options.get("time", False)
STORE_RESPONSE = options.get("response", False)
VERBOSITY = int(options.get("verbosity", 1))
# EACH_URL = options.get('each', 100000)
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, [""]) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
conf_urls = {}
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [""])
except Exception, e:
print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e)))
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
# Get function name and add it to the hash of URLConf urls
func_name = hasattr(func, "__name__") and func.__name__ or repr(func)
conf_urls[regex] = ["func.__module__", func_name]
def _test_url(self, urlpatterns):
deconstructed_urls = extract_views_from_urlpatterns(urlpatterns)
added_kwargs = {}
# we need to format urls with proper ids
url_ctx = self.get_url_path_ctx()
if url_ctx:
self.response_data = {
url.format(**url_ctx): data
for url, data in self.response_data.items()
}
for (view, regex, namespace, name) in deconstructed_urls:
request_data = self.request_data.get(name, {}).copy()
for key in list(re.compile(regex).groupindex.keys()):
if key in list(request_data.keys()):
added_kwargs[key] = request_data[key]
continue
if key not in self.default_kwargs:
raise Exception(
'URL argument not in test kwargs. Please add `%s`' %
key)
added_kwargs[key] = self.default_kwargs[key]
path = reverse(name, kwargs=added_kwargs)
self.assertResponse(path=path, name=name)
added_kwargs = {}
def handle(self, *args, **options):
if args:
appname, = args
style = color_style()
if settings.ADMIN_FOR:
settings_modules = [
__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR
]
else:
settings_modules = [settings]
views = []
for settings_mod in settings_modules:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
view_functions = extract_views_from_urlpatterns(
urlconf.urlpatterns)
for (func, regex) in view_functions:
views.append(
"%(url)s\t%(module)s.%(name)s" % {
'name': style.MODULE_NAME(func.__name__),
'module': style.MODULE(func.__module__),
'url': style.URL(simplify_regex(regex))
})
return "\n".join([v for v in views])
def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
depth = int(options.get('depth', 3))
auth = _parse_auth(options.get('auth'))
if verbosity > 1:
log_level = logging.DEBUG
elif verbosity:
log_level = logging.INFO
else:
log_level = logging.WARN
crawl_logger = logging.getLogger('crawler')
crawl_logger.setLevel(logging.DEBUG)
crawl_logger.propagate = 0
log_stats = LogStatsHandler()
crawl_logger.addHandler(log_stats)
console = logging.StreamHandler()
console.setLevel(log_level)
console.setFormatter(
logging.Formatter(
"%(name)s [%(levelname)s] %(module)s: %(message)s"))
crawl_logger.addHandler(console)
if len(args) > 1:
raise CommandError('Only one start url is currently supported.')
else:
start_url = args[0] if args else '/'
if getattr(settings, 'ADMIN_FOR', None):
settings_modules = [
__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR
]
else:
settings_modules = [settings]
conf_urls = {}
# Build the list URLs to test from urlpatterns:
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
logging.exception("Error occurred while trying to load %s: %s",
settings_mod.ROOT_URLCONF, str(e))
continue
view_functions = extract_views_from_urlpatterns(
urlconf.urlpatterns)
for (func, regex, namespace, name) in view_functions:
#Get function name and add it to the hash of URLConf urls
func_name = hasattr(func,
'__name__') and func.__name__ or repr(func)
conf_urls[regex] = ['func.__module__', func_name]
def get_url_name_choices(self):
urlconf = import_module(settings.ROOT_URLCONF)
ignored = self.Meta.model.URL_NAME_IGNORED or ()
choices = [(':'.join(ns + [name]), path) for func, path, ns, name in
extract_views_from_urlpatterns(urlconf.urlpatterns)
if name and not ('(?P' in path or '<' in path)]
return [(name, '/%s (%s)' % (url, name)) for name, url in choices
if not ignored or not any(re.match(i, name) for i in ignored)]
def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
depth = int(options.get('depth', 3))
auth = _parse_auth(options.get('auth'))
if verbosity == 3:
log_level = 1
elif verbosity == 2:
log_level = logging.DEBUG
elif verbosity:
log_level = logging.INFO
else:
log_level = logging.WARN
crawl_logger = logging.getLogger('crawler')
crawl_logger.setLevel(logging.DEBUG)
crawl_logger.propagate = 0
log_stats = LogStatsHandler()
crawl_logger.addHandler(log_stats)
console = logging.StreamHandler()
console.setLevel(log_level)
console.setFormatter(logging.Formatter("%(name)s [%(levelname)s] %(module)s: %(message)s"))
crawl_logger.addHandler(console)
if len(args) > 1:
raise CommandError('Only one start url is currently supported.')
else:
start_url = args[0] if args else '/'
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
conf_urls = {}
# Build the list URLs to test from urlpatterns:
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
logging.exception("Error occurred while trying to load %s: %s", settings_mod.ROOT_URLCONF, str(e))
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
#Get function name and add it to the hash of URLConf urls
func_name = hasattr(func, '__name__') and func.__name__ or repr(func)
conf_urls[regex] = ['func.__module__', func_name]
def get_aio_routes(patterns=None):
"""Walk the URL patterns to find any coroutine views."""
if patterns is None:
urlconf = import_module(settings.ROOT_URLCONF)
patterns = urlconf.urlpatterns
routes = []
view_functions = extract_views_from_urlpatterns(patterns)
for (func, regex, namespace, name) in view_functions:
if asyncio.iscoroutinefunction(func) or inspect.isgeneratorfunction(func):
routes.append(DjangoRegexRoute('*', func, name, regex))
return routes
def get_aio_routes(patterns=None):
"""Walk the URL patterns to find any coroutine views."""
if patterns is None:
urlconf = import_module(settings.ROOT_URLCONF)
patterns = urlconf.urlpatterns
routes = []
view_functions = extract_views_from_urlpatterns(patterns)
for (func, regex, namespace, name) in view_functions:
if asyncio.iscoroutinefunction(func) or inspect.isgeneratorfunction(
func):
routes.append(DjangoRegexRoute('*', func, name, regex))
return routes
def check_policy_for_all_routes():
errors = []
urlconf = import_module(settings.ROOT_URLCONF)
# extract_views_from_urlpatterns is an undocumented internal from the
# contrib 'admindocs' module that happens to be useful for demo purposes.
for (func, regex, namespace,
name) in extract_views_from_urlpatterns(urlconf.urlpatterns):
if not regex.startswith('policies/'):
continue
if not has_security_policy_applied(func):
errors.append((
f"{func.__module__}.{func.__name__} needs to have a security policy applied",
regex), )
return errors
def _test_url(self, urlpatterns):
deconstructed_urls = extract_views_from_urlpatterns(urlpatterns)
added_kwargs = {}
for (view, regex, namespace, name) in deconstructed_urls:
request_data = self.request_data.get(name, {}).copy()
for key in list(re.compile(regex).groupindex.keys()):
if key in list(request_data.keys()):
added_kwargs[key] = request_data[key]
continue
if key not in self.default_kwargs:
raise Exception('URL argument not in test kwargs. Please add `%s`' % key)
added_kwargs[key] = self.default_kwargs[key]
path = reverse(name, kwargs=added_kwargs)
self.assertResponse(path=path, name=name)
added_kwargs = {}
def get_service_endpoint_urls(urlconfig=None, default_value='1'):
"""
This function finds all endpoint urls in a service.
Args:
urlconfig: A django url config module to use. Defaults to settings.ROOT_URLCONF
default_value: A string value to replace all url parameters with
Returns:
A list of urls with path parameters (i.e. <pk>) replaced with default_value
"""
if not urlconfig:
urlconfig = getattr(settings, 'ROOT_URLCONF')
try:
urlconfig_mod = import_module(urlconfig)
except Exception as ex:
raise Exception(
"Unable to import url config module. Url Config: {0}. Message: {1}"
.format(urlconfig, ex.message))
extracted_views = extract_views_from_urlpatterns(urlconfig_mod.urlpatterns)
views_regex_url_patterns = [item[1] for item in extracted_views]
simplified_regex_url_patterns = [
simplify_regex(pattern) for pattern in views_regex_url_patterns
]
# Strip out urls we don't need to test.
result_urls = []
pattern = re.compile(r'<\w+>')
for url in simplified_regex_url_patterns:
if url.find('<format>') != -1:
continue
if url == u'/':
continue
if url == u'/health':
continue
parameters = pattern.findall(url)
for param in parameters:
url = url.replace(param, default_value)
result_urls.append(url)
return result_urls
def _get_urls(self):
urls = []
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception as e:
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for view_function in view_functions:
urls.append(simplify_regex(view_function[1]))
return urls
def _get_urls(self):
urls = []
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception as e:
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for view_function in view_functions:
urls.append(simplify_regex(view_function[1]))
return urls
def get_service_endpoint_urls(urlconfig=None, default_value='1'):
"""
This function finds all endpoint urls in a service.
Args:
urlconfig: A django url config module to use. Defaults to settings.ROOT_URLCONF
default_value: A string value to replace all url parameters with
Returns:
A list of urls with path parameters (i.e. <pk>) replaced with default_value
"""
if not urlconfig:
urlconfig = getattr(settings, 'ROOT_URLCONF')
try:
urlconfig_mod = import_module(urlconfig)
except Exception as ex:
raise Exception(
"Unable to import url config module. Url Config: {0}. Message: {1}".format(urlconfig, ex.message))
extracted_views = extract_views_from_urlpatterns(urlconfig_mod.urlpatterns)
views_regex_url_patterns = [item[1] for item in extracted_views]
simplified_regex_url_patterns = [simplify_regex(pattern) for pattern in views_regex_url_patterns]
# Strip out urls we don't need to test.
result_urls = []
pattern = re.compile(r'<\w+>')
for url in simplified_regex_url_patterns:
if url.find('<format>') != -1:
continue
if url == u'/':
continue
if url == u'/health':
continue
parameters = pattern.findall(url)
for param in parameters:
url = url.replace(param, default_value)
result_urls.append(url)
return result_urls
def handle(self, *args, **options):
if args:
appname, = args
style = color_style()
if settings.ADMIN_FOR:
settings_modules = [
__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR
]
else:
settings_modules = [settings]
views = []
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
if options.get('traceback', None):
import traceback
self.stderr.write(traceback.format_exc() + "\n")
self.stderr.write(
style.ERROR(
"Error occurred while trying to load %s: %s\n" %
(settings_mod.ROOT_URLCONF, str(e))))
continue
view_functions = \
extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
func_name = hasattr(func, '__name__') and \
func.__name__ or repr(func)
views.append(
"%(url)s\t%(module)s.%(name)s" % {
'name':
style.MODULE_NAME(func_name),
'module':
style.MODULE(getattr(func, '__module__',
'<no module>')),
'url':
style.URL(simplify_regex(regex))
})
def handle(self, *args, **options):
if args:
appname, = args
style = color_style()
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
views = []
for settings_mod in settings_modules:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
views.append("%(url)s\t%(module)s.%(name)s" % {'name': style.MODULE_NAME(func.__name__),
'module': style.MODULE(func.__module__),
'url': style.URL(simplify_regex(regex))})
return "\n".join([v for v in views])
def get_urls(urlconf_name):
urlconf = import_module(urlconf_name)
all_urls = extract_views_from_urlpatterns(urlconf.urlpatterns)
for mounted_url in all_urls:
viewfunc = mounted_url[0]
regex = mounted_url[1]
try:
namespace = mounted_url[2]
except IndexError:
namespace = None
if namespace is None:
namespace = []
try:
name = mounted_url[3]
except IndexError:
name = None
modname = viewfunc.__module__
viewname = getattr(viewfunc, '__name__', viewfunc.__class__.__name__)
simple_url = simplify_regex(regex)
named_groups = all_params_re.findall(simple_url)
all_groups = (replace_simple_regex(var, index)
for index, var in enumerate(named_groups, start=1))
textsearch = set()
textsearch.update(modname.lower().split('.'))
textsearch.add(viewname.lower())
if name:
textsearch.add(name.lower())
textsearch.update(x.lower() for x in simple_url.split('/')
if "<" not in x and x.strip() != '')
yield {
'textsearch': " ".join(textsearch),
'textdata': textsearch,
'module': modname,
'view': viewname,
'url': simple_url,
'namespace': namespace,
'namespace_string': ":".join(namespace),
'name': name,
'named_url_kwargs': tuple(all_groups),
}
def get_urls(urlconf_name):
urlconf = import_module(urlconf_name)
all_urls = extract_views_from_urlpatterns(urlconf.urlpatterns)
for mounted_url in all_urls:
viewfunc = mounted_url[0]
regex = mounted_url[1]
try:
namespace = mounted_url[2]
except IndexError:
namespace = None
if namespace is None:
namespace = []
try:
name = mounted_url[3]
except IndexError:
name = None
modname = viewfunc.__module__
viewname = getattr(viewfunc, '__name__', viewfunc.__class__.__name__)
simple_url = simplify_regex(regex)
named_groups = all_params_re.findall(simple_url)
all_groups = (replace_simple_regex(var, index)
for index, var in enumerate(named_groups, start=1))
textsearch = set()
textsearch.update(modname.lower().split('.'))
textsearch.add(viewname.lower())
if name:
textsearch.add(name.lower())
textsearch.update(x.lower() for x in simple_url.split('/') if "<" not in x and x.strip() != '')
yield {
'textsearch': " ".join(textsearch),
'textdata': textsearch,
'module': modname,
'view': viewname,
'url': simple_url,
'namespace': namespace,
'namespace_string': ":".join(namespace),
'name': name,
'named_url_kwargs': tuple(all_groups),
}
def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
if settings.ADMIN_FOR:
settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR]
else:
settings_modules = [settings]
conf_urls = {}
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception, e:
print ("Error occurred while trying to load %s: %s" % (settings_mod.ROOT_URLCONF, str(e)))
continue
view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns)
for (func, regex) in view_functions:
#Get function name and add it to the hash of URLConf urls
func_name = hasattr(func, '__name__') and func.__name__ or repr(func)
conf_urls[regex] = ['func.__module__', func_name]
def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None):
# taken from 16a842b3795ca78a5918538ab6b9f1afbd718f72
views = []
for p in urlpatterns:
if hasattr(p, 'url_patterns'):
try:
patterns = p.url_patterns
except ImportError:
continue
views.extend(
extract_views_from_urlpatterns(
patterns, base + p.regex.pattern, (namespace or []) +
(p.namespace and [p.namespace] or [])))
elif hasattr(p, 'callback'):
try:
views.append((p.callback, base + p.regex.pattern,
namespace, p.name))
except ViewDoesNotExist:
continue
else:
raise TypeError(
_("%s does not appear to be a urlpattern object") % p)
return views
def extract_views_from_urlpatterns(urlpatterns, base='', namespace=None):
# taken from 16a842b3795ca78a5918538ab6b9f1afbd718f72
views = []
for p in urlpatterns:
if hasattr(p, 'url_patterns'):
try:
patterns = p.url_patterns
except ImportError:
continue
views.extend(extract_views_from_urlpatterns(
patterns,
base + p.regex.pattern,
(namespace or []) + (p.namespace and [p.namespace] or [])
))
elif hasattr(p, 'callback'):
try:
views.append((p.callback, base + p.regex.pattern,
namespace, p.name))
except ViewDoesNotExist:
continue
else:
raise TypeError(_("%s does not appear to be a urlpattern object") % p)
return views
def _test_url(self, urlpatterns):
deconstructed_urls = extract_views_from_urlpatterns(urlpatterns)
added_kwargs = {}
# we need to format urls with proper ids
url_ctx = self.get_url_path_ctx()
if url_ctx:
self.response_data = {
url.format(**url_ctx): data for url, data in self.response_data.items()
}
for (view, regex, namespace, name) in deconstructed_urls:
request_data = self.request_data.get(name, {}).copy()
for key in list(re.compile(regex).groupindex.keys()):
if key in list(request_data.keys()):
added_kwargs[key] = request_data[key]
continue
if key not in self.default_kwargs:
raise Exception('URL argument not in test kwargs. Please add `%s`' % key)
added_kwargs[key] = self.default_kwargs[key]
path = reverse(name, kwargs=added_kwargs)
self.assertResponse(path=path, name=name)
added_kwargs = {}
def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
depth = int(options.get('depth', 3))
auth = _parse_auth(options.get('auth'))
if verbosity > 1:
log_level = logging.DEBUG
elif verbosity:
log_level = logging.INFO
else:
log_level = logging.WARN
crawl_logger = logging.getLogger('crawler')
crawl_logger.setLevel(logging.DEBUG)
crawl_logger.propagate = 0
log_stats = LogStatsHandler()
crawl_logger.addHandler(log_stats)
console = logging.StreamHandler()
console.setLevel(log_level)
console.setFormatter(
logging.Formatter(
"%(name)s [%(levelname)s] %(module)s: %(message)s"))
crawl_logger.addHandler(console)
if len(args) > 1:
raise CommandError('Only one start url is currently supported.')
else:
start_url = args[0] if args else '/'
if getattr(settings, 'ADMIN_FOR', None):
settings_modules = [
__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR
]
else:
settings_modules = [settings]
conf_urls = {}
# Build the list URLs to test from urlpatterns:
for settings_mod in settings_modules:
try:
urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, [''])
except Exception as e:
logging.exception("Error occurred while trying to load %s: %s",
settings_mod.ROOT_URLCONF, str(e))
continue
view_functions = extract_views_from_urlpatterns(
urlconf.urlpatterns)
for (func, regex, namespace, name) in view_functions:
# Get function name and add it to the hash of URLConf urls
func_name = hasattr(func,
'__name__') and func.__name__ or repr(func)
conf_urls[regex] = ['func.__module__', func_name]
c = Crawler(
start_url,
conf_urls=conf_urls,
verbosity=verbosity,
output_dir=options.get("output_dir"),
ascend=not options.get("no_parent"),
auth=auth,
)
# Load plugins:
for p in options['plugins']:
# This nested try is somewhat unsightly but allows easy Pythonic
# usage ("--enable-plugin=tidy") instead of Java-esque
# "--enable-plugin=test_utils.crawler.plugins.tidy"
try:
try:
plugin_module = __import__(p)
except ImportError:
if not "." in p:
plugin_module = __import__(
"test_utils.crawler.plugins.%s" % p,
fromlist=["test_utils.crawler.plugins"])
else:
raise
c.plugins.append(plugin_module.PLUGIN())
except (ImportError, AttributeError) as e:
crawl_logger.critical("Unable to load plugin %s: %s", p, e)
sys.exit(3)
c.run(max_depth=depth)
# We'll exit with a non-zero status if we had any errors
max_log_level = max(log_stats.stats.keys())
if max_log_level >= logging.ERROR:
sys.exit(2)
elif max_log_level >= logging.WARNING:
sys.exit(1)
else:
sys.exit(0)
