def compile(self, paths, force=False):
for index, input_path in enumerate(paths):
for compiler in self.compilers:
compiler = compiler(verbose=self.verbose, storage=self.storage)
if compiler.match_file(input_path):
output_path = self.output_path(input_path,
compiler.output_extension)
paths[index] = output_path
try:
infile = finders.find(input_path)
outfile = finders.find(output_path)
if outfile is None:
outfile = self.output_path(
infile, compiler.output_extension)
outdated = True
else:
outdated = self.is_outdated(
input_path, output_path)
compiler.compile_file(infile,
outfile,
outdated=outdated,
force=force)
except CompilerError:
if not self.storage.exists(
output_path) or not settings.PIPELINE:
raise
return paths
def compile_paths(self, paths, compilers, force=False):
for index, input_path in enumerate(paths):
for compiler in compilers:
compiler = compiler(self.verbose)
if path_is_url(input_path):
input_path = get_remote_path(input_path)
if compiler.match_file(input_path):
output_path = compiler.output_path(input_path)
paths[index] = output_path
try:
infile = finders.find(input_path)
outfile = finders.find(output_path)
outdated = self.is_outdated(input_path, output_path)
compiler.compile_file(infile,
outfile,
outdated=outdated,
force=force)
except CompilerError:
if not self.storage.exists(
output_path) or not settings.PIPELINE:
raise
input_path = output_path
return paths
def compile(self, paths, force=False):
for index, input_path in enumerate(paths):
for compiler in self.compilers:
compiler = compiler(self.verbose)
if compiler.match_file(input_path):
output_path = self.output_path(input_path, compiler.output_extension)
paths[index] = output_path
try:
infile = finders.find(input_path)
outfile = finders.find(output_path)
if outfile is None:
outfile = self.output_path(infile, compiler.output_extension)
outdated = True
else:
outdated = self.is_outdated(input_path, output_path)
if input_path in settings.PIPELINE_ALWAYS_RECOMPILE:
force = True
LOG.info('Compiling %s to %s, outdated=%s, force=%s',
infile, outfile, outdated, force)
compiler.compile_file(infile, outfile, outdated=outdated, force=force)
except CompilerError:
if not self.storage.exists(output_path) or not settings.PIPELINE:
raise
return paths
def read_file(self, path):
"""Read file content in binary mode"""
path = finders.find(path)
file = open(path, "rb")
content = file.read()
file.close()
return content
def compile(self, paths, force=False):
for index, input_path in enumerate(paths):
for compiler in self.compilers:
compiler = compiler(self.verbose)
if compiler.match_file(input_path):
output_path = self.output_path(input_path, compiler.output_extension)
paths[index] = output_path
try:
infile = finders.find(input_path)
outfile = finders.find(output_path)
outdated = self.is_outdated(input_path, output_path)
compiler.compile_file(infile, outfile, outdated=outdated, force=force)
except CompilerError:
if not self.storage.exists(output_path) or not settings.PIPELINE:
raise
return paths
def serve(request, path, document_root=None, insecure=False, **kwargs):
"""
Serve static files below a given point in the directory structure or
from locations inferred from the staticfiles finders.
To use, put a URL pattern such as::
(r'^(?P<path>.*)$', 'staticfiles.views.serve')
in your URLconf.
It automatically falls back to django.views.static
"""
if not settings.DEBUG and not insecure:
raise ImproperlyConfigured(
"The staticfiles view can only be used in "
"debug mode or if the the --insecure "
"option of 'runserver' is used"
)
normalized_path = posixpath.normpath(urllib.unquote(path)).lstrip("/")
absolute_path = finders.find(normalized_path)
if not absolute_path:
raise Http404("'%s' could not be found" % path)
document_root, path = os.path.split(absolute_path)
return static.serve(request, path, document_root=document_root, **kwargs)
def read_file(self, path):
"""Read file content in binary mode"""
path = finders.find(path)
file = self.storage.open(path, 'rb')
content = file.read()
file.close()
return content
def serve(request, path, show_indexes=False, insecure=False):
"""
Serve static files below a given point in the directory structure or
from locations inferred from the static files finders.
To use, put a URL pattern such as::
(r'^(?P<path>.*)$', 'staticfiles.views.serve')
in your URLconf.
If you provide the ``document_root`` parameter, the file won't be looked
up with the staticfiles finders, but in the given filesystem path, e.g.::
(r'^(?P<path>.*)$', 'staticfiles.views.serve', {'document_root' : '/path/to/my/files/'})
You may also set ``show_indexes`` to ``True`` if you'd like to serve a
basic index of the directory. This index view will use the
template hardcoded below, but if you'd like to override it, you can create
a template called ``static/directory_index.html``.
"""
if not settings.DEBUG and not insecure:
raise ImproperlyConfigured("The view to serve static files can only "
"be used if the DEBUG setting is True or "
"the --insecure option of 'runserver' is "
"used")
absolute_path = finders.find(path)
if not absolute_path:
raise Http404('"%s" could not be found' % path)
document_root, path = os.path.split(absolute_path)
return django_serve(request, path=path, document_root=document_root,
show_indexes=show_indexes)
def __add_manholes(self, layers, styles):
"Add manhole layer and styles."
# Select the manholes that are part of this sewerage.
manholes = Manhole.objects.filter(sewerage__pk=self.id)
# Define a style.
style = mapnik.Style()
# Style the `normal` manholes.
rule = mapnik.Rule()
rule.filter = mapnik.Filter("[sink] != 1")
symbol = mapnik.PointSymbolizer()
symbol.allow_overlap = True
rule.symbols.append(symbol)
style.rules.append(rule)
# Style the sink.
rule = mapnik.Rule()
rule.filter = mapnik.Filter("[sink] = 1")
symbol = mapnik.PointSymbolizer(
str(finders.find("lizard_riool/sink.png")), "png", 8, 8
)
symbol.allow_overlap = True
rule.symbols.append(symbol)
style.rules.append(rule)
# Add labels.
rule = mapnik.Rule()
rule.max_scale = 1700
symbol = mapnik.TextSymbolizer(
'code', 'DejaVu Sans Book', 10, mapnik.Color('black')
)
symbol.allow_overlap = True
symbol.label_placement = mapnik.label_placement.POINT_PLACEMENT
symbol.vertical_alignment = mapnik.vertical_alignment.TOP
symbol.displacement(0, -5) # slightly above
rule.symbols.append(symbol)
style.rules.append(rule)
# Setup datasource.
params = default_database_params()
params['table'] = "({}) data".format(manholes.query)
datasource = mapnik.PostGIS(**params)
# Define layer.
layer = mapnik.Layer('manholeLayer')
layer.datasource = datasource
layer.maxzoom = 35000
layer.styles.append('manholeStyle')
layers.append(layer)
styles['manholeStyle'] = style
def serve(request, path, document_root=None, insecure=False, **kwargs):
"""
Serve static files below a given point in the directory structure or
from locations inferred from the staticfiles finders.
To use, put a URL pattern such as::
(r'^(?P<path>.*)$', 'staticfiles.views.serve')
in your URLconf.
It automatically falls back to django.views.static
"""
if not settings.DEBUG and not insecure:
raise ImproperlyConfigured("The staticfiles view can only be used in "
"debug mode or if the the --insecure "
"option of 'runserver' is used")
normalized_path = posixpath.normpath(urllib.unquote(path)).lstrip('/')
absolute_path = finders.find(normalized_path)
if not absolute_path:
if path.endswith('/') or path == '':
raise Http404("Directory indexes are not allowed here.")
raise Http404("'%s' could not be found" % path)
document_root, path = os.path.split(absolute_path)
return static.serve(request, path, document_root=document_root, **kwargs)
def find(self, path, all=False):
"""
Work out the uncached name of the file and look that up instead
"""
try:
start, _, extn = path.rsplit('.', 2)
path = '.'.join((start, extn))
return find(path, all=all)
except ValueError:
return []
def replace_static_url(original, prefix, quote, rest):
"""
Replace a single matched url.
"""
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif (not static_asset_path) \
and course_id \
and modulestore().get_modulestore_type(course_id) != ModuleStoreEnum.Type.xml:
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the edx-platform repo (e.g. JS associated with an xmodule)
exists_in_staticfiles_storage = False
try:
exists_in_staticfiles_storage = staticfiles_storage.exists(
rest)
except Exception as err:
log.warning(
u"staticfiles_storage couldn't find path {0}: {1}".format(
rest, unicode(err)))
if exists_in_staticfiles_storage:
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url_with_course_id(
rest, course_id)
if AssetLocator.CANONICAL_NAMESPACE in url:
url = url.replace('block@', 'block/', 1)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((static_asset_path or data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning(
"staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def handle_label(self, path, **options):
verbosity = int(options.get('verbosity', 1))
result = finders.find(path, all=options['all'])
if result:
if not isinstance(result, (list, tuple)):
result = [result]
output = '\n '.join((os.path.realpath(path) for path in result))
sys.stdout.write("Found %r here:\n %s\n" % (path, output))
else:
if verbosity >= 1:
sys.stdout.write("No matching file found for %r.\n" % path)
def replace_static_url(match):
original = match.group(0)
prefix = match.group('prefix')
quote = match.group('quote')
rest = match.group('rest')
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif (not static_asset_path) and course_id and modulestore(
).get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
exists_in_staticfiles_storage = False
try:
exists_in_staticfiles_storage = staticfiles_storage.exists(
rest)
except Exception as err:
log.warning(
"staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
if exists_in_staticfiles_storage:
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url_with_course_id(
rest, course_id)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((static_asset_path or data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning(
"staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def embeddable(self, path, variant):
"""Is the asset embeddable ?"""
name, ext = os.path.splitext(path)
font = ext in FONT_EXTS
if not variant:
return False
if not (re.search(EMBEDDABLE, path) and finders.find(path)):
return False
if not ext in EMBED_EXTS:
return False
if not (font or len(self.encoded_content(path)) < MAX_IMAGE_SIZE):
return False
return True
def replace_static_url(original, prefix, quote, rest):
"""
Replace a single matched url.
"""
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif (not static_asset_path) \
and course_id \
and modulestore().get_modulestore_type(course_id) != ModuleStoreEnum.Type.xml:
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the edx-platform repo (e.g. JS associated with an xmodule)
exists_in_staticfiles_storage = False
try:
exists_in_staticfiles_storage = staticfiles_storage.exists(rest)
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
if exists_in_staticfiles_storage:
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
if AssetLocator.CANONICAL_NAMESPACE in url:
url = url.replace('block@', 'block/', 1)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((static_asset_path or data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def replace_static_url(match):
original = match.group(0)
prefix = match.group("prefix")
quote = match.group("quote")
rest = match.group("rest")
# Don't mess with things that end in '?raw'
if rest.endswith("?raw"):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif (
(not static_asset_path)
and course_id
and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE
):
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the edx-platform repo (e.g. JS associated with an xmodule)
exists_in_staticfiles_storage = False
try:
exists_in_staticfiles_storage = staticfiles_storage.exists(rest)
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(rest, str(err)))
if exists_in_staticfiles_storage:
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((static_asset_path or data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def handle_label(self, path, **options):
verbosity = int(options.get('verbosity', 1))
result = finders.find(path, all=options['all'])
path = smart_unicode(path)
if result:
if not isinstance(result, (list, tuple)):
result = [result]
output = u'\n '.join(
(smart_unicode(os.path.realpath(path)) for path in result))
sys.stdout.write(
smart_str(u"Found '%s' here:\n %s\n" % (path, output)))
else:
if verbosity >= 1:
sys.stderr.write(
smart_str("No matching file found for '%s'.\n" % path))
def compile(self, paths):
for index, path in enumerate(paths):
for compiler in self.compilers:
compiler = compiler(self.verbose)
if compiler.match_file(path):
new_path = self.output_path(path, compiler.output_extension)
paths[index] = new_path
try:
content = self.read_file(path)
compiled_content = compiler.compile_file(content, finders.find(path))
self.save_file(new_path, compiled_content)
except CompilerError:
if not self.storage.exists(new_path) or not settings.PIPELINE:
raise
return paths
def handle_label(self, path, **options):
verbosity = int(options.get('verbosity', 1))
result = finders.find(path, all=options['all'])
path = smart_unicode(path)
if result:
if not isinstance(result, (list, tuple)):
result = [result]
output = u'\n '.join(
(smart_unicode(os.path.realpath(path)) for path in result))
sys.stdout.write(
smart_str(u"Found '%s' here:\n %s\n" % (path, output)))
else:
if verbosity >= 1:
sys.stderr.write(
smart_str("No matching file found for '%s'.\n" % path))
def compile(self, paths, force=False):
for index, path in enumerate(paths):
for compiler in self.compilers:
compiler = compiler(self.verbose)
if compiler.match_file(path):
new_path = self.output_path(path, compiler.output_extension)
paths[index] = new_path
if not force and not self.is_outdated(path, new_path):
continue
try:
content = self.read_file(path)
compiled_content = compiler.compile_file(content, finders.find(path))
self.save_file(new_path, compiled_content)
except CompilerError:
if not self.storage.exists(new_path) or not settings.PIPELINE:
raise
return paths
def replace_static_url(match):
original = match.group(0)
prefix = match.group('prefix')
quote = match.group('quote')
rest = match.group('rest')
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not
# None, then use studio style urls
elif course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the mitx repo (e.g. JS
# associated with an xmodule)
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url(
rest, course_namespace)
# Otherwise, look the file up in staticfiles_storage, and append the
# data directory if needed
else:
course_path = "/".join((data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add
# manually data directory
except Exception as err:
log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def findfile(self, url):
url = urllib.unquote(url).split("?", 1)[0]
url = os.path.normpath(url.lstrip('/'))
prefix = [ settings.STATIC_URL, settings.MEDIA_URL ]
if settings.STATIC_URL[0] == '/':
prefix.append(settings.STATIC_URL[1:])
if settings.MEDIA_URL[0] == '/':
prefix.append(settings.MEDIA_URL[1:])
path = None
for p in prefix:
if p in url:
length = url.find(p) + len(p)
path = url[length:]
if path:
path = url
else:
raise Exception('Could not open the file %s' % url)
return finders.find(path)
def read_file(self, path):
path = finders.find(path)
file = open(path, 'rb')
content = file.read()
file.close()
return content