def download_submission(request, submission, revision):
create_hit(request, submission, extra_info="download")
if submission.sub_type == 'snippet':
response = HttpResponse(mimetype="application/x-python")
fname = submission.slug.replace('-', '_') + '.py'
response["Content-Disposition"] = "attachment; filename=%s" % fname
source = Site.objects.get_current().domain + \
submission.get_absolute_url()
response.write('# Source: ' + source + '\n\n' + revision.item_code)
return response
if submission.sub_type == 'package':
zip_dir = os.path.join(settings.SPC['ZIP_staging'], 'download')
ensuredir(zip_dir)
response = HttpResponse(mimetype="attachment; application/zip")
zip_name = '%s-%d-%d.zip' % (submission.slug, submission.id,
revision.rev_id_human)
response['Content-Disposition'] = 'filename=%s' % zip_name
full_zip_file = os.path.join(zip_dir, zip_name)
if not os.path.exists(full_zip_file):
# Set the repo's state to the state when that particular revision
# existed
out = submission.fileset.checkout_revision(revision.hash_id)
if out:
logger.info('Checked out revision "%s" for rev.id=%d' % \
(revision.hash_id, revision.id))
else:
logger.error('Could not checked out revision "%s" for '
'rev.id=%d' % (revision.hash_id, revision.id))
return page_404_error(request, ('Could not create the ZIP '
'file. This error has been '
'reported.'))
zip_f = zipfile.ZipFile(full_zip_file, "w", zipfile.ZIP_DEFLATED)
src_dir = os.path.join(settings.SPC['storage_dir'],
submission.fileset.repo_path)
for path, dirs, files in os.walk(src_dir):
for name in files:
file_name = os.path.join(path, name)
file_h = open(file_name, "r")
zip_f.write(file_name, file_name.partition(src_dir)[2])
file_h.close()
for file_h in zip_f.filelist:
file_h.create_system = 0
zip_f.close()
# Return the repo checkout back to the most recent revision
out = submission.fileset.checkout_revision(submission.\
last_revision.hash_id)
# Return the ZIP file
zip_data = open(full_zip_file, "rb")
response.write(zip_data.read())
zip_data.close()
return response
def __create_repo(self):
"""
Create empty repository for the fileset object
"""
if not isinstance(self.object.entry.fileset, FileSet):
raise TypeError('FileSet object not passed to fileset attribute')
repo_path = self.object.entry.fileset.repo_path
ensuredir(os.path.join(settings.SPC['storage_dir'], repo_path))
repo = self.object.entry.fileset.create_empty()
# log info
logger.info('SubmissionStorage:: Created an empty repository: '
'Path [dir=%s] Revision [pk=%d] User [pk=%d]'
% (repo_path, self.object.pk, self.object.created_by.pk))
return repo
def call_sphinx_to_compile(working_dir):
"""
Changes to the ``working_dir`` directory and compiles the RST files to
pickle files, according to settings in the conf.py file.
Returns nothing, but logs if an error occurred.
"""
working_dir = os.path.abspath(working_dir)
build_dir = os.path.join(working_dir, '_build')
ensuredir(build_dir)
status = StringIO()
warning = StringIO()
try:
app = Sphinx(srcdir=working_dir,
confdir=working_dir,
outdir=os.path.join(build_dir, 'pickle'),
doctreedir=os.path.join(build_dir, 'doctrees'),
buildername='pickle',
status=status,
warning=warning,
freshenv=True,
warningiserror=False,
tags=[])
# We need to access some settings while inside Sphinx
# This is the easiest way to get them there
app.env.config.SPC = settings.SPC
app.env.config.SPC['__Screenshot__'] = Screenshot
# Call the ``pickle`` builder
app.build(force_all=True)
except SphinxError as error:
if warning.tell():
warning.seek(0)
for line in warning.readlines():
logger.warn('COMMENT: ' + line)
msg = (('Sphinx error occurred when compiling comment '
'(error type = %s): %s' % (error.category, str(error))))
logger.error(msg)
raise SphinxError(msg)
if app.statuscode != 0:
logger.error("Non-zero status code when compiling.")
def call_sphinx_to_compile(working_dir):
"""
Changes to the ``working_dir`` directory and compiles the RST files to
pickle files, according to settings in the conf.py file.
Returns nothing, but logs if an error occurred.
"""
build_dir = os.path.abspath(working_dir + os.sep + "_build")
ensuredir(build_dir)
status = StringIO()
warning = StringIO()
try:
app = Sphinx(
srcdir=working_dir,
confdir=working_dir,
outdir=build_dir + os.sep + "pickle",
doctreedir=build_dir + os.sep + "doctrees",
buildername="pickle",
status=status,
warning=warning,
freshenv=True,
warningiserror=False,
tags=[],
)
# We need to access some settings while inside Sphinx
# This is the easiest way to get them there
app.env.config.SPC = settings.SPC
app.env.config.SPC["__Screenshot__"] = Screenshot
# Call the ``pickle`` builder
app.build(force_all=True)
except SphinxError as error:
if warning.tell():
warning.seek(0)
for line in warning.readlines():
logger.warn("COMMENT: " + line)
msg = "Sphinx error occurred when compiling comment " "(error type = %s): %s" % (error.category, str(error))
logger.error(msg)
raise SphinxError(msg)
if app.statuscode != 0:
logger.error("Non-zero status code when compiling.")
def setup_compile_dir(compile_dir):
"""
Setup a directory for Sphinx compilation.
We need certain files in place to compile the comments Copy the
settings, image extension, and an __init__.py to the appropriate
places. The original copy of the ``conf.py`` file, found in the
current directory (copy it to comment destination)
"""
if Site._meta.installed:
site = Site.objects.get_current().domain
else:
site = ''
module_dir = os.path.dirname(__file__)
ext_dir = os.path.abspath(os.path.join(compile_dir, 'ext'))
ensuredir(compile_dir)
ensuredir(ext_dir)
conf_template_file = os.path.join(module_dir, 'sphinx-conf.py')
conf_file = os.path.join(compile_dir, 'conf.py')
with file(conf_template_file, 'r') as f:
conf_template = template.Template(f.read())
if settings.MEDIA_URL.startswith('http'):
conf = conf_template.render(template.Context({'FULL_MEDIA_URL':
settings.MEDIA_URL}))
else:
conf = conf_template.render(template.Context({'FULL_MEDIA_URL':
site + settings.MEDIA_URL}))
with file(conf_file, 'w') as f:
f.write(conf)
fn = os.path.join(module_dir, 'images.py')
shutil.copyfile(fn, os.path.join(compile_dir, 'ext', 'images.py'))
fn = os.path.join(module_dir, '__init__.py')
shutil.copyfile(fn, os.path.join(compile_dir, 'ext', '__init__.py'))
def setup_compile_dir(compile_dir):
"""
Setup a directory for Sphinx compilation.
We need certain files in place to compile the comments Copy the
settings, image extension, and an __init__.py to the appropriate
places. The original copy of the ``conf.py`` file, found in the
current directory (copy it to comment destination)
"""
if Site._meta.installed:
site = Site.objects.get_current().domain
else:
site = ''
module_dir = os.path.dirname(__file__)
ext_dir = os.path.abspath(os.path.join(compile_dir, 'ext'))
ensuredir(compile_dir)
ensuredir(ext_dir)
conf_template_file = os.path.join(module_dir, 'sphinx-conf.py')
conf_file = os.path.join(compile_dir, 'conf.py')
with file(conf_template_file, 'r') as f:
conf_template = template.Template(f.read())
if settings.MEDIA_URL.startswith('http'):
conf = conf_template.render(
template.Context({'FULL_MEDIA_URL': settings.MEDIA_URL}))
else:
conf = conf_template.render(
template.Context({'FULL_MEDIA_URL': site + settings.MEDIA_URL}))
with file(conf_file, 'w') as f:
f.write(conf)
fn = os.path.join(module_dir, 'images.py')
shutil.copyfile(fn, os.path.join(compile_dir, 'ext', 'images.py'))
fn = os.path.join(module_dir, '__init__.py')
shutil.copyfile(fn, os.path.join(compile_dir, 'ext', '__init__.py'))
def compile_rest_to_html(raw_rest):
""" Compiles the RST string, ``raw_RST`, to HTML. Performs no
further checking on the RST string.
If it is a comment, then we don't modify the HTML with extra class info.
But we do filter comments to disable hyperlinks.
Also copy over generated MATH media to the correct directory on the server.
"""
def sanitize_raw_rest(raw_rest):
"""
Performs any sanitizing of the user's input.
Currently performs:
* Converts string to utf-8 encoding
* Converts '\\' to '\\\\': i.e. single slash converted to double-slash,
because Sphinx converts is back to a single slash
* Removes hyperlinks to remote images
Alternative way to prevent csv-table directive, raw directive,
include directive: if using docutils.
For more, see: http://docutils.sourceforge.net/docs/user/config.html#file-insertion-enabled
"""
raw_rest = raw_rest.encode('utf-8')
# Required: so that things like \( a = \frac{b}{c} \) works as
# expected, otherwise users have to write \\ a = \\frac{b}{c} \\)
raw_rest = raw_rest.replace('\\', '\\\\')
# Replace tabs with 4 spaces: so that source code listings don't get
# the default 8 spaces that Sphinx/docutils use.
raw_rest = raw_rest.replace('\t', ' ')
raw_rest = raw_rest.split('\r\n')
# Strip '.. raw::'' directive
raw = re.compile(r'^(\s*)..(\s*)raw(\s*)::(\s*)')
math_role = re.compile(r':math:`(.+?)`')
math_env = re.compile(r'^\s*.. math::*')
math_lines = []
for idx, line in enumerate(raw_rest):
if raw.match(line):
raw_rest[idx] = ''
if math_env.match(line):
math_lines.append(idx)
# Fix double \\\\ in :math:` ... `
outline = ''
last_pos = 0
for math_str in math_role.finditer(line):
outline += line[last_pos:math_str.start()+7]
outline += math_str.group(1).replace('\\\\', '\\')
outline += '`'
last_pos = math_str.end()
outline += line[last_pos:]
raw_rest[idx] = outline
# Are there math environments we need to remove "\\" from?
for _, line_num in enumerate(math_lines):
# From http://ucomment.org
prefix_re = re.compile('^\s*')
prefix_match = prefix_re.match(raw_rest[line_num])
prefix = prefix_match.group()
# Search down to find where the math environment ends
finished = False
next_line = ''
end_line = line_num + 1
for idx, line in enumerate(raw_rest[line_num+1:]):
end_line += 1
bias = idx + 2
if line.strip() == '':
# Keep looking further down
for _, next_line in enumerate(raw_rest[line_num+bias:]):
if next_line.strip() != '':
finished = True
break
if finished:
next_prefix = prefix_re.match(next_line.rstrip('\n')).group()
# Break if a non-blank line has the same, or lower indent
# level than the environment's level (``prefix``)
if len(next_prefix.expandtabs()) <= len(prefix.expandtabs()):
break
else:
finished = False
# All done: replace the \\\\ with \\
for i, math_str in enumerate(raw_rest[line_num:end_line]):
math_str = math_str.replace('\\\\', '\\')
raw_rest[i+line_num] = math_str
# Remove hyperlinks to remote items: e.g. .. include:: http://badstuff.com
NO_REMOTE = ['literalinclude', 'include', 'csv-table']
for item in NO_REMOTE:
remote_re = re.compile(r'^(\s*)..(\s*)' + item + r'(\s*)::(\s*)http')
for idx, line in enumerate(raw_rest):
if remote_re.match(line):
raw_rest[idx] = ''
return '\r\n'.join(raw_rest)
def call_sphinx_to_compile(working_dir):
"""
Changes to the ``working_dir`` directory and compiles the RST files to
pickle files, according to settings in the conf.py file.
Returns nothing, but logs if an error occurred.
"""
working_dir = os.path.abspath(working_dir)
build_dir = os.path.join(working_dir, '_build')
ensuredir(build_dir)
status = StringIO()
warning = StringIO()
try:
app = Sphinx(srcdir=working_dir, confdir=working_dir,
outdir = os.path.join(build_dir, 'pickle'),
doctreedir = os.path.join(build_dir, 'doctrees'),
buildername = 'pickle',
status = status,
warning = warning,
freshenv = True,
warningiserror = False,
tags = [])
# We need to access some settings while inside Sphinx
# This is the easiest way to get them there
app.env.config.SPC = settings.SPC
app.env.config.SPC['__Screenshot__'] = Screenshot
# Call the ``pickle`` builder
app.build(force_all=True)
except SphinxError as error:
if warning.tell():
warning.seek(0)
for line in warning.readlines():
logger.warn('COMMENT: ' + line)
msg = (('Sphinx error occurred when compiling comment '
'(error type = %s): %s' % (error.category, str(error))))
logger.error(msg)
raise SphinxError(msg)
if app.statuscode != 0:
logger.error("Non-zero status code when compiling.")
# Create a directory where Sphinx will compile the ReST
ensuredir(settings.SPC['comment_compile_dir'])
compile_dir = tempfile.mkdtemp(dir=settings.SPC['comment_compile_dir'])
try:
setup_compile_dir(compile_dir)
logger.debug('SPHINX: ' + raw_rest)
modified_rest = sanitize_raw_rest(raw_rest)
with open(os.path.join(compile_dir, 'index.rst'), 'w') as fh:
fh.write(modified_rest)
# Compile the comment
call_sphinx_to_compile(compile_dir)
pickle_f = os.path.join(compile_dir, '_build', 'pickle', 'index.fpickle')
with open(pickle_f, 'rb') as fhand:
obj = pickle.load(fhand)
finally:
shutil.rmtree(compile_dir)
return obj['body'].encode('utf-8')
def save(self, *args, **kwargs):
""" Override the model's saving function to ensure the repo dir is
created. """
utils.ensuredir(os.path.join(storage_dir, self.repo_path))
super(FileSet, self).save(*args, **kwargs)
def test_dvcs(self):
pass
backends = ['hg']
for backend in backends:
# Start from scratch every time
shutil.rmtree(self.tempdir)
utils.ensuredir(self.tempdir)
utils.ensuredir(self.local_path)
utils.ensuredir(self.remote_path)
f = open(self.remote_path + 'index.rst', 'w')
f.writelines(['Header\n','======\n', '\n', 'Paragraph 1\n', '\n',
'Paragraph 2\n', '\n', 'Paragraph 3\n'])
f.close()
remote_repo = dvcs.DVCSRepo(backend, self.remote_path)
remote_repo.add('.')
remote_repo.commit('Initial commit', user="******")
remote_hash = remote_repo.get_revision_info()
## Create, add and commit to the remote repo
## Verify that we cannot expect to query the remote repo:
##self.assertRaises(dvcs.DVCSError, dvcs.get_revision_info, remote=True)
local_repo = remote_repo.clone(self.local_path)
local_hash = local_repo.check_out(rev='tip')
self.assertEqual(local_hash, remote_hash)
# Now, in the local repo, make some changes to test
# Add a comment for paragraph 2; commit
f = open(self.local_path + 'index.rst', 'w')
f.writelines(['Header\n','======\n', '\n', 'Paragraph 1\n', '\n',
'Paragraph 2\n', '\n', '.. ucomment:: aaaaaa: 11,\n',
'\n', 'Paragraph 3\n'])
f.close()
self.assertRaises(dvcs.DVCSError, local_repo.push)
local_repo.set_remote(self.remote_path)
rev1 = local_repo.update_commit_and_push_updates(message='Para 2')
# Check out an old revision to modify, rather than the latest revision
new_hash = local_repo.check_out(rev=local_hash)
self.assertEqual(new_hash, local_hash)
# Now add a comment to paragraph 3, but from the initial revision
f = open(self.local_path + 'index.rst', 'w')
f.writelines(['Header\n','======\n', '\n', 'Paragraph 1\n', '\n',
'Paragraph 2\n', '\n', 'Paragraph 3\n', '\n',
'.. ucomment:: bbbbbb: 22,\n'])
f.close()
rev2 = local_repo.update_commit_and_push_updates(message='Para 3')
# Add a comment above on the local repo, again starting from old version
hex_str = local_repo.check_out(rev=local_hash)
# Now add a comment to paragraph 1
f = open(self.local_path + 'index.rst', 'w')
f.writelines(['Header\n','======\n', '\n', 'Paragraph 1\n', '\n',
'.. ucomment:: cccccc: 33,\n', '\n', 'Paragraph 2\n',
'\n', 'Paragraph 3\n'])
f.close()
rev3 = local_repo.update_commit_and_push_updates(message='Para 1')
f = open(self.local_path + 'index.rst', 'r')
lines = f.readlines()
f.close()
final_result = ['Header\n', '======\n', '\n', 'Paragraph 1\n',
'\n', '.. ucomment:: cccccc: 33,\n', '\n',
'Paragraph 2\n', '\n',
'.. ucomment:: aaaaaa: 11,\n', '\n',
'Paragraph 3\n', '\n',
'.. ucomment:: bbbbbb: 22,\n']
self.assertEqual(lines, final_result)
# Now test the code in dvcs.pull_update_and_merge(...).
# Handles the basic case when the author makes changes (they are pushed
# to the remote repo) and they should be imported imported into the
# local repo without requiring a merge.
final_result.insert(3, 'A new paragraph.\n')
final_result.insert(4, '\n')
with open(self.remote_path + 'index.rst', 'w') as f_handle:
f_handle.writelines(final_result)
remote_repo.commit(message='Remote update.')
local_repo.pull_update_and_merge()
with open(self.local_path + 'index.rst', 'r') as f_handle:
local_lines = f_handle.readlines()
self.assertEqual(local_lines, final_result)
def compile_rest_to_html(raw_rest):
""" Compiles the RST string, ``raw_RST`, to HTML. Performs no
further checking on the RST string.
If it is a comment, then we don't modify the HTML with extra class info.
But we do filter comments to disable hyperlinks.
Also copy over generated MATH media to the correct directory on the server.
"""
def sanitize_raw_rest(raw_rest):
"""
Performs any sanitizing of the user's input.
Currently performs:
* Converts string to utf-8 encoding
* Converts '\\' to '\\\\': i.e. single slash converted to double-slash,
because Sphinx converts is back to a single slash
* Removes hyperlinks to remote images
Alternative way to prevent csv-table directive, raw directive,
include directive: if using docutils.
For more, see: http://docutils.sourceforge.net/docs/user/config.html#file-insertion-enabled
"""
raw_rest = raw_rest.encode('utf-8')
# Required: so that things like \( a = \frac{b}{c} \) works as
# expected, otherwise users have to write \\ a = \\frac{b}{c} \\)
raw_rest = raw_rest.replace('\\', '\\\\')
# Replace tabs with 4 spaces: so that source code listings don't get
# the default 8 spaces that Sphinx/docutils use.
raw_rest = raw_rest.replace('\t', ' ')
raw_rest = raw_rest.split('\r\n')
# Strip '.. raw::'' directive
raw = re.compile(r'^(\s*)..(\s*)raw(\s*)::(\s*)')
math_role = re.compile(r':math:`(.+?)`')
math_env = re.compile(r'^\s*.. math::*')
math_lines = []
for idx, line in enumerate(raw_rest):
if raw.match(line):
raw_rest[idx] = ''
if math_env.match(line):
math_lines.append(idx)
# Fix double \\\\ in :math:` ... `
outline = ''
last_pos = 0
for math_str in math_role.finditer(line):
outline += line[last_pos:math_str.start() + 7]
outline += math_str.group(1).replace('\\\\', '\\')
outline += '`'
last_pos = math_str.end()
outline += line[last_pos:]
raw_rest[idx] = outline
# Are there math environments we need to remove "\\" from?
for _, line_num in enumerate(math_lines):
# From http://ucomment.org
prefix_re = re.compile('^\s*')
prefix_match = prefix_re.match(raw_rest[line_num])
prefix = prefix_match.group()
# Search down to find where the math environment ends
finished = False
next_line = ''
end_line = line_num + 1
for idx, line in enumerate(raw_rest[line_num + 1:]):
end_line += 1
bias = idx + 2
if line.strip() == '':
# Keep looking further down
for _, next_line in enumerate(raw_rest[line_num + bias:]):
if next_line.strip() != '':
finished = True
break
if finished:
next_prefix = prefix_re.match(
next_line.rstrip('\n')).group()
# Break if a non-blank line has the same, or lower indent
# level than the environment's level (``prefix``)
if len(next_prefix.expandtabs()) <= len(
prefix.expandtabs()):
break
else:
finished = False
# All done: replace the \\\\ with \\
for i, math_str in enumerate(raw_rest[line_num:end_line]):
math_str = math_str.replace('\\\\', '\\')
raw_rest[i + line_num] = math_str
# Remove hyperlinks to remote items: e.g. .. include:: http://badstuff.com
NO_REMOTE = ['literalinclude', 'include', 'csv-table']
for item in NO_REMOTE:
remote_re = re.compile(r'^(\s*)..(\s*)' + item +
r'(\s*)::(\s*)http')
for idx, line in enumerate(raw_rest):
if remote_re.match(line):
raw_rest[idx] = ''
return '\r\n'.join(raw_rest)
def call_sphinx_to_compile(working_dir):
"""
Changes to the ``working_dir`` directory and compiles the RST files to
pickle files, according to settings in the conf.py file.
Returns nothing, but logs if an error occurred.
"""
working_dir = os.path.abspath(working_dir)
build_dir = os.path.join(working_dir, '_build')
ensuredir(build_dir)
status = StringIO()
warning = StringIO()
try:
app = Sphinx(srcdir=working_dir,
confdir=working_dir,
outdir=os.path.join(build_dir, 'pickle'),
doctreedir=os.path.join(build_dir, 'doctrees'),
buildername='pickle',
status=status,
warning=warning,
freshenv=True,
warningiserror=False,
tags=[])
# We need to access some settings while inside Sphinx
# This is the easiest way to get them there
app.env.config.SPC = settings.SPC
app.env.config.SPC['__Screenshot__'] = Screenshot
# Call the ``pickle`` builder
app.build(force_all=True)
except SphinxError as error:
if warning.tell():
warning.seek(0)
for line in warning.readlines():
logger.warn('COMMENT: ' + line)
msg = (('Sphinx error occurred when compiling comment '
'(error type = %s): %s' % (error.category, str(error))))
logger.error(msg)
raise SphinxError(msg)
if app.statuscode != 0:
logger.error("Non-zero status code when compiling.")
# Create a directory where Sphinx will compile the ReST
ensuredir(settings.SPC['comment_compile_dir'])
compile_dir = tempfile.mkdtemp(dir=settings.SPC['comment_compile_dir'])
try:
setup_compile_dir(compile_dir)
logger.debug('SPHINX: ' + raw_rest)
modified_rest = sanitize_raw_rest(raw_rest)
with open(os.path.join(compile_dir, 'index.rst'), 'w') as fh:
fh.write(modified_rest)
# Compile the comment
call_sphinx_to_compile(compile_dir)
pickle_f = os.path.join(compile_dir, '_build', 'pickle',
'index.fpickle')
with open(pickle_f, 'rb') as fhand:
obj = pickle.load(fhand)
finally:
shutil.rmtree(compile_dir)
return obj['body'].encode('utf-8')
from scipy_central.utils import ensuredir
from scipy_central.screenshot.models import Screenshot
# We need certain files in place to compile the comments
# Copy the settings, image extension, and an __init__.py to the appropriate
# places. The original copy of the ``conf.py`` file, found in the current
# directory (copy it to comment destination)
working_dir = settings.SPC['comment_compile_dir']
if Site._meta.installed:
site = Site.objects.get_current().domain
else:
site = ''
ext_dir = os.path.abspath(working_dir + os.sep + 'ext')
ensuredir(working_dir)
ensuredir(ext_dir)
cf = os.path.abspath(__file__ + os.sep + os.path.pardir) + \
os.sep + 'sphinx-conf.py'
conf_file = settings.SPC['comment_compile_dir'] + os.sep + 'conf.py'
shutil.copyfile(cf, conf_file)
with file(conf_file, 'r') as conf_file_raw:
fc = conf_file_raw.read()
resp = template.Template(fc)
if settings.MEDIA_URL.startswith('http'):
conf_file_text = resp.render(template.Context({'FULL_MEDIA_URL': \
settings.MEDIA_URL}))
else:
conf_file_text = resp.render(template.Context({'FULL_MEDIA_URL': \
def create_or_edit_submission_revision(request, item, is_displayed,
user, submission=None, commit=False):
"""
Creates a new ``Submission`` (only if not given) and ``Revision``
instances. Returns these in a tuple.
"""
# NOTE: the ``user`` will always be a valid entry in our database. Code
# posted by users that have not yet validated themselves is not displayed
# until they do so.
new_submission = False
if submission is None:
# A new submission
new_submission = True
submission = models.Submission.objects.create_without_commit(
created_by=user, sub_type=item.cleaned_data['sub_type'])
sub = submission
# Process any tags
tag_list = get_and_create_tags(item.cleaned_data['sub_tags'])
# Create a ``Revision`` instance. Must always have a ``title``,
# ``created_by``, and ``description`` fields; the rest are set according
# to the submission type, ``sub.sub_type``
hash_id = ''
if sub.sub_type == 'link':
sub_license = None
item_url = item.cleaned_data['item_url']
item_code = None
elif sub.sub_type == 'snippet':
sub_license = item.cleaned_data['sub_license']
item_url = None
item_code = item.cleaned_data['snippet_code']
elif sub.sub_type == 'package':
sub_license = item.cleaned_data['sub_license']
item_url = None
item_code = None
# Handle the ZIP file more completely only when the user commits.
# ZIP file has been validated: OK to save it to the server
# However, this might be the second time around, so skip saving it
# (happens after preview, or if user resumes editing submission)
if not hasattr(request.FILES['package_file'], 'skip_validation'):
zip_f = models.ZipFile(raw_zip_file=request.FILES['package_file'],
zip_hash=request.POST.get('package_hash', ''))
zip_f.save()
# Convert the raw ReST description to HTML using Sphinx: could include
# math, paragraphs, <tt>, bold, italics, bullets, hyperlinks, etc.
description_html = compile_rest_to_html(item.cleaned_data['description'])
item_highlighted_code = highlight_code(item.cleaned_data.get(\
'snippet_code', None))
rev = models.Revision.objects.create_without_commit(
entry=sub,
title=item.cleaned_data['title'],
created_by=user,
sub_license=sub_license,
description=item.cleaned_data['description'],
description_html=description_html,
hash_id=hash_id,
item_url=item_url,
item_code=item_code,
item_highlighted_code=item_highlighted_code,
is_displayed=is_displayed,
)
user_url = settings.SPC['short_URL_root'] + 'user/' + str(user.id)
if commit:
# Save the submission, then the revision. If we are editing a
# previous submission, then do not save the submission
# (because the Submission object has fields that will never
# change once it has been first created).
if new_submission:
# Sets the primary key
sub.save()
rev.entry_id = sub.id
if not is_displayed:
rev.validation_hash = create_validation_code(rev)
rev.save()
# Storage location: if we do save files it will be here
datenow = datetime.datetime.now()
year, month = datenow.strftime('%Y'), datenow.strftime('%m')
repo_path = os.path.join(year, month, '%06d'% sub.id)
full_repo_path = os.path.join(settings.SPC['storage_dir'], repo_path)
if sub.sub_type == 'package':
# Save the uploaded file to the server. At this point we are sure
# it's a valid ZIP file, has no malicious filenames, and can be
# unpacked to the hard drive. See validation in ``forms.py``.
if os.path.exists(full_repo_path) and sub.fileset.repo_path == \
repo_path:
# Make a temporary directory and copy the existing package
# repository to that location
temp_dir = tempfile.mkdtemp(prefix='tmp_spc_')
src = os.path.join(full_repo_path, '.' + \
settings.SPC['revisioning_backend'])
shutil.move(src, temp_dir)
shutil.rmtree(full_repo_path, ignore_errors=True)
else:
temp_dir = None
# Create/ensure destination directory exists
ensuredir(full_repo_path)
# Copy ZIP file
zip_file = request.FILES['package_file']
dst = os.path.join(full_repo_path, zip_file.name)
src = os.path.join(settings.SPC['ZIP_staging'], zip_file.name)
shutil.copyfile(src, dst)
# os.remove(src) Keep the original ZIP file, for now
# Remove the entry from the database
zip_hash = request.POST.get('package_hash', '')
zip_objs = models.ZipFile.objects.filter(zip_hash=zip_hash)
if zip_objs:
zip_objs[0].delete()
# Unzip file and commit contents to the repo
zip_f = zipfile.ZipFile(dst, 'r')
zip_f.extractall(full_repo_path)
zip_f.close()
os.remove(dst) # but delete the copy
# Delete common RCS directories that might have been in the ZIP
for path, dirs, files in os.walk(full_repo_path):
if os.path.split(path)[1] in settings.SPC['common_rcs_dirs']:
shutil.rmtree(path, ignore_errors=True)
if temp_dir:
src = os.path.join(temp_dir, '.' + \
settings.SPC['revisioning_backend'])
dst = os.path.join(full_repo_path , '.' + \
settings.SPC['revisioning_backend'])
try:
os.rename(src, dst)
except os.error, e:
# For cases when /tmp is on a different filesystem
# (usually production servers)
import errno
if e.errno == errno.EXDEV:
shutil.copytree(src, dst, symlinks=True)
shutil.rmtree(src)
else:
raise
shutil.rmtree(temp_dir, ignore_errors=True)
repo = sub.fileset.get_repo()
else:
# Create the repo
sub.fileset = FileSet.objects.create(repo_path=repo_path)
repo = sub.fileset.create_empty()
sub.save()
# Then add all files from the ZIP file to the repo. Add directories
# at a time rather than file-by-file.
for path, dirs, files in os.walk(full_repo_path):
if os.path.split(path)[1] == '.' + \
settings.SPC['revisioning_backend']:
for entry in dirs[:]:
dirs.remove(entry)
continue
all_files = []
for name in files:
all_files.append(os.path.join(path, name))
if all_files:
repo.add(patterns=all_files, ignore_errors=True)
# Add "DESCRIPTION.txt"
descrip_name = os.path.join(full_repo_path, 'DESCRIPTION.txt')
descrip_file = file(descrip_name, 'w')
descrip_file.write(rev.description)
descrip_file.close()
sub.fileset.add_file(descrip_name, user=user_url,
commit_msg=('Added/updated files from web-uploaded '
'ZIP file. Added DESCRIPTION.txt also.'))
if sub.sub_type == 'snippet':
fname = rev.slug.replace('-', '_') + '.py'
if new_submission:
# Create a new repository for the files
sub.fileset = FileSet.objects.create(repo_path=repo_path)
sub.save()
commit_msg = ('Add "%s" to the repo '
'based on the web submission by user "%s"') %\
(fname, user_url)
else:
commit_msg = ('Update of file(s) in the repo '
'based on the web submission by user "%s"') %\
(user_url)
sub.fileset.add_file_from_string(fname,
request.POST['snippet_code'],
user=user_url,
commit_msg=commit_msg)
if sub.sub_type in ['snippet', 'package']:
license_file = settings.SPC['license_filename']
license_text = get_license_text(rev)
sub.fileset.add_file_from_string(license_file, license_text,
user="******",
commit_msg="SPC: added/updated license file" )
rev.hash_id = sub.fileset.get_hash()
# Once you have the revision you can add tags through the intermediate
# model instance (which tracks the user that added the tag and when).
for tag in tag_list:
tag_intermediate = models.TagCreation(created_by=user,
revision=rev,
tag=tag)
tag_intermediate.save()
logger.debug('User "%s" added tag "%s" to rev.id=%d' % (
user.profile.slug,
str(tag), rev.id))
# Update the search index so that the tags are included in the search
rev.save()
# log the new submission and revision
logger.info('New %s: %s [id=%d] and revision id=%d' % (
sub.sub_type,
item.cleaned_data['title'],
sub.id,
rev.id))
def save(self, *args, **kwargs):
""" Override the model's saving function to create the slug """
ensuredir(os.path.join(settings.MEDIA_ROOT, settings.SPC['ZIP_staging']))
super(ZipFile, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
""" Override the model's saving function to ensure the repo dir is
created. """
utils.ensuredir(os.path.join(storage_dir, self.repo_path))
super(FileSet, self).save(*args, **kwargs)
