def render_markdown_from_file(f, **markdown_kwargs):
"""Render Markdown text from a file stream to HTML."""
s = StringIO()
markdownFromFile(input=f, output=s, **markdown_kwargs)
html = s.getvalue()
s.close()
return html
def render_markdown_from_file(f, **markdown_kwargs):
"""Render Markdown text from a file stream to HTML."""
s = StringIO()
markdownFromFile(input=f, output=s, **markdown_kwargs)
html = s.getvalue()
s.close()
return html
def render_markdown_from_file(f):
"""Renders Markdown text to HTML.
The Markdown text will be sanitized to prevent injecting custom HTML.
It will also enable a few plugins for code highlighting and sane lists.
"""
s = StringIO()
markdownFromFile(input=f, output=s, **MARKDOWN_KWARGS)
html = s.getvalue()
s.close()
return html
def render_markdown_from_file(f):
"""Renders Markdown text to HTML.
The Markdown text will be sanitized to prevent injecting custom HTML.
It will also enable a few plugins for code highlighting and sane lists.
"""
s = StringIO()
markdownFromFile(input=f, output=s, **MARKDOWN_KWARGS)
html = s.getvalue()
s.close()
return html
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
try:
from django_evolution.management.commands.evolve import \
Command as Evolution
except ImportError:
raise InstallExtensionError(
"Unable to migrate the extension's database tables. Django "
"Evolution is not installed.")
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0,
interactive=False,
execute=True,
hint=False,
compile_sql=False,
purge=False,
database=False,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e,
output,
exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def parse(self):
"""
Parses the diff, returning a list of File objects representing each
file in the diff.
"""
logging.debug("DiffParser.parse: Beginning parse of diff, size = %s",
len(self.data))
preamble = StringIO()
self.files = []
parsed_file = None
i = 0
# Go through each line in the diff, looking for diff headers.
while i < len(self.lines):
next_linenum, new_file = self.parse_change_header(i)
if new_file:
# This line is the start of a new file diff.
#
# First, finalize the last one.
if self.files:
self.files[-1].finalize()
parsed_file = new_file
# We need to prepend the preamble, if we have one.
parsed_file.prepend_data(preamble.getvalue())
preamble.close()
preamble = StringIO()
self.files.append(parsed_file)
i = next_linenum
else:
if parsed_file:
i = self.parse_diff_line(i, parsed_file)
else:
preamble.write(self.lines[i])
preamble.write(b'\n')
i += 1
if self.files:
self.files[-1].finalize()
preamble.close()
logging.debug("DiffParser.parse: Finished parsing diff.")
return self.files
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
try:
from django_evolution.management.commands.evolve import \
Command as Evolution
except ImportError:
raise InstallExtensionError(
"Unable to migrate the extension's database tables. Django "
"Evolution is not installed.")
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0, interactive=False,
execute=True, hint=False,
compile_sql=False, purge=False,
database=False,
stdout=stream, stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e, output, exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
if django_evolution is None:
# Django Evolution isn't installed. Extensions with evolutions
# are not supported.
return
try:
stream = StringIO()
call_command('evolve',
verbosity=0,
interactive=False,
execute=True,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e,
output,
exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
if django_evolution is None:
# Django Evolution isn't installed. Extensions with evolutions
# are not supported.
return
try:
stream = StringIO()
call_command('evolve',
verbosity=0,
interactive=False,
execute=True,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logger.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logger.exception('Error evolving extension models: %s: %s',
e, output)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def arch_month_mbox(request, list_name, year, month_name):
"""
The messages must be rebuilt before being added to the mbox file, including
headers and the textual content, making sure to escape email addresses.
"""
return HttpResponse("Not implemented yet.",
content_type="text/plain", status=500)
mlist = get_list_by_name(list_name, request.get_host())
month = month_name_to_num(month_name)
year = int(year)
begin_date = datetime.datetime(year, month, 1)
if month != 12:
end_month = month + 1
else:
end_month = 1
end_date = datetime.datetime(year, end_month, 1)
messages = Email.objects.filter(
mailinglist=mlist, date__gte=begin_date, date__lte=end_date
).order_by("date")
mboxfile, mboxfilepath = tempfile.mkstemp(
prefix="hyperkitty-", suffix=".mbox.gz")
os.close(mboxfile)
mbox = mailbox.mbox(mboxfilepath)
for msg in messages:
mbox.add(msg.full)
mbox.close()
content = StringIO()
zipped_content = gzip.GzipFile(fileobj=content)
with gzip.GzipFile(fileobj=content, mode="wb") as zipped_content:
with open(mboxfilepath, "rb") as mboxfile:
zipped_content.write(mboxfile.read())
response = HttpResponse(content.getvalue())
content.close()
response['Content-Type'] = "application/mbox+gz"
response['Content-Disposition'] = 'attachment; filename=%d-%s.txt.gz' \
% (year, month_name)
response['Content-Length'] = len(response.content)
os.remove(mboxfilepath)
return response
class FileStream(object):
"""File stream for streaming reponses
This buffer intended for use as an argument to StreamingHTTPResponse
and also as a file for TarFile to write into.
Files are read in by chunks and written to this buffer through TarFile.
When there is content to be read from the buffer, it is taken up by
StreamingHTTPResponse and the buffer is cleared to prevent storing large
chunks of data in memory.
"""
def __init__(self):
self.buffer = StringIO()
self.offset = 0
def write(self, s):
"""Write ``s`` to the buffer and adjust the offset."""
self.buffer.write(s)
self.offset += len(s)
def tell(self):
"""Return the current position of the buffer."""
return self.offset
def close(self):
"""Close the buffer."""
self.buffer.close()
def pop(self):
"""Return the current contents of the buffer then clear it."""
s = self.buffer.getvalue()
self.buffer.close()
self.buffer = StringIO()
return s
def parse(self):
"""
Parses the diff, returning a list of File objects representing each
file in the diff.
"""
self.files = []
i = 0
preamble = StringIO()
while i < len(self.lines):
next_i, file_info, new_diff = self._parse_diff(i)
if file_info:
if self.files:
self.files[-1].finalize()
self._ensure_file_has_required_fields(file_info)
file_info.prepend_data(preamble.getvalue())
preamble.close()
preamble = StringIO()
self.files.append(file_info)
elif new_diff:
# We found a diff, but it was empty and has no file entry.
# Reset the preamble.
preamble.close()
preamble = StringIO()
else:
preamble.write(self.lines[i])
preamble.write(b'\n')
i = next_i
try:
if self.files:
self.files[-1].finalize()
elif preamble.getvalue().strip() != b'':
# This is probably not an actual git diff file.
raise DiffParserError('This does not appear to be a git diff',
0)
finally:
preamble.close()
return self.files
def _install_extension_media_internal(self, ext_class):
"""Installs extension data.
Performs any installation necessary for an extension.
If the extension has a legacy htdocs/ directory for static media
files, they will be installed into MEDIA_ROOT/ext/, and a warning
will be logged.
If the extension has a modern static/ directory, they will be
installed into STATIC_ROOT/ext/.
"""
ext_htdocs_path = ext_class.info.installed_htdocs_path
ext_htdocs_path_exists = os.path.exists(ext_htdocs_path)
if ext_htdocs_path_exists:
# First, get rid of the old htdocs contents, so we can start
# fresh.
shutil.rmtree(ext_htdocs_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'):
# This is an older extension that doesn't use the static file
# support. Log a deprecation notice and then install the files.
logging.warning('The %s extension uses the deprecated "htdocs" '
'directory for static files. It should be updated '
'to use a "static" directory instead.'
% ext_class.info.name)
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__, 'htdocs')
shutil.copytree(extracted_path, ext_htdocs_path, symlinks=True)
# We only want to install static media on a non-DEBUG install.
# Otherwise, we run the risk of creating a new 'static' directory and
# causing Django to look up all static files (not just from
# extensions) from there instead of from their source locations.
if not settings.DEBUG:
ext_static_path = ext_class.info.installed_static_path
ext_static_path_exists = os.path.exists(ext_static_path)
if ext_static_path_exists:
# Also get rid of the old static contents.
shutil.rmtree(ext_static_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'static'):
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__,
'static')
shutil.copytree(extracted_path, ext_static_path, symlinks=True)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
# Now let's build any tables that this extension might need
self._add_to_installed_apps(ext_class)
# Call syncdb to create the new tables
loading.cache.loaded = False
call_command('syncdb', verbosity=0, interactive=False)
# Run evolve to do any table modification
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0, interactive=False,
execute=True, hint=False,
compile_sql=False, purge=False,
database=False,
stdout=stream, stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e, output, exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
# Remove this again, since we only needed it for syncdb and
# evolve. _init_extension will add it again later in
# the install.
self._remove_from_installed_apps(ext_class)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
class ParsedDiffFile(object):
"""A parsed file from a diff.
This stores information on a single file represented in a diff, including
the contents of that file's diff, as parsed by :py:class:`DiffParser` or
one of its subclasses.
Parsers should set the attributes on this based on the contents of the
diff, and should add any data found in the diff.
This class is meant to be used internally and by subclasses of
:py:class:`DiffParser`.
"""
def __init__(self):
"""Initialize the parsed file information."""
self.origFile = None
self.newFile = None
self.origInfo = None
self.newInfo = None
self.origChangesetId = None
self.binary = False
self.deleted = False
self.moved = False
self.copied = False
self.insert_count = 0
self.delete_count = 0
self._data_io = StringIO()
self._data = None
@property
def data(self):
"""The data for this diff.
This must be accessed after :py:meth:`finalize` has been called.
"""
if self._data is None:
raise ValueError('ParsedDiffFile.data cannot be accessed until '
'finalize() is called.')
return self._data
def finalize(self):
"""Finalize the parsed diff.
This makes the diff data available to consumers and closes the buffer
for writing.
"""
self._data = self._data_io.getvalue()
self._data_io.close()
def prepend_data(self, data):
"""Prepend data to the buffer.
Args:
data (bytes):
The data to prepend.
"""
if data:
new_data_io = StringIO()
new_data_io.write(data)
new_data_io.write(self._data_io.getvalue())
self._data_io.close()
self._data_io = new_data_io
def append_data(self, data):
"""Append data to the buffer.
Args:
data (bytes):
The data to append.
"""
if data:
self._data_io.write(data)
