def get_display(submission, revision, filename): """ Determines how to display a filetype, given its name """ fname = filename[-1] mime_guess = mimetypes.guess_type(fname)[0] mime_type, mime_file = mime_guess.split('/') # Set the repo to the correct revision repo = submission.fileset.checkout_revision(revision.hash_id) src = os.path.join(repo.local_dir, os.sep.join(filename)) if str(mime_type).startswith('image'): # We only dislay certain image types VALID_IMAGE_TYPES = ['gif', 'jpeg', 'png', 'bmp'] if mime_file in VALID_IMAGE_TYPES: disp_type = 'image' # Copy image over to media location; we must make a copy, incase # a later user views a different revision of the document disp_obj = os.path.normpath(force_unicode(datetime.datetime.now()\ .strftime(smart_str(settings.SPC['resized_image_dir']))))\ + os.sep + fname dst = os.path.join(settings.SPC['storage_dir'], disp_obj) idx = 1 while os.path.exists(dst): disp_obj = disp_obj.split(fname)[0] + '%s_%d.%s' % \ (fname.lower().split('.'+mime_file)[0], idx, mime_file) dst = os.path.join(settings.SPC['storage_dir'], disp_obj) idx += 1 # Finally, copy the file across to the web storage area shutil.copy2(src, dst) return disp_type, disp_obj if not repo: # Something went wrong when checking out the repo logger.error('Could not checked out revision "%s" for ' 'rev.id=%d' % (revision.hash_id, revision.id)) return 'none', None if str(mime_type).startswith('text'): # Read the first 10kb to send to the lexer guessing mechanism if os.path.exists(src): fh = open(src, 'rb') file_content = fh.readlines(10*1024) amount_read = fh.tell() fh.close() else: return 'none', None try: lexer = guess_lexer_for_filename(fname.lower(), ''.join(file_content)) except ClassNotFound: pass else: disp_type = 'html' # Only re-read the file if we didn't read it all the first time if os.path.getsize(src) == amount_read: file_content = ''.join(file_content) else: fh = open(src, 'rb') file_content = fh.read() fh.close() # TODO(KGD): consider wrapping long text lines for text files # Return the highlighted code, if we know the lexer return disp_type, highlight_code(file_content, lexer=lexer.mimetypes[0]) # All other file types are assumed to be binary disp_type = 'binary' #disp_obj = link to file (add this capability to ``FileSet``) return disp_type, disp_obj
def create_or_edit_submission_revision(request, item, is_displayed, user, submission=None, commit=False): """ Creates a new ``Submission`` (only if not given) and ``Revision`` instances. Returns these in a tuple. """ # NOTE: the ``user`` will always be a valid entry in our database. Code # posted by users that have not yet validated themselves is not displayed # until they do so. new_submission = False if submission is None: # A new submission new_submission = True submission = models.Submission.objects.create_without_commit( created_by=user, sub_type=item.cleaned_data['sub_type']) sub = submission # Process any tags tag_list = get_and_create_tags(item.cleaned_data['sub_tags']) # Create a ``Revision`` instance. Must always have a ``title``, # ``created_by``, and ``description`` fields; the rest are set according # to the submission type, ``sub.sub_type`` hash_id = '' if sub.sub_type == 'link': sub_license = None item_url = item.cleaned_data['item_url'] item_code = None elif sub.sub_type == 'snippet': sub_license = item.cleaned_data['sub_license'] item_url = None item_code = item.cleaned_data['snippet_code'] elif sub.sub_type == 'package': sub_license = item.cleaned_data['sub_license'] item_url = None item_code = None # Handle the ZIP file more completely only when the user commits. # ZIP file has been validated: OK to save it to the server # However, this might be the second time around, so skip saving it # (happens after preview, or if user resumes editing submission) if not hasattr(request.FILES['package_file'], 'skip_validation'): zip_f = models.ZipFile(raw_zip_file=request.FILES['package_file'], zip_hash=request.POST.get('package_hash', '')) zip_f.save() # Convert the raw ReST description to HTML using Sphinx: could include # math, paragraphs, <tt>, bold, italics, bullets, hyperlinks, etc. description_html = compile_rest_to_html(item.cleaned_data['description']) item_highlighted_code = highlight_code(item.cleaned_data.get(\ 'snippet_code', None)) rev = models.Revision.objects.create_without_commit( entry=sub, title=item.cleaned_data['title'], created_by=user, sub_license=sub_license, description=item.cleaned_data['description'], description_html=description_html, hash_id=hash_id, item_url=item_url, item_code=item_code, item_highlighted_code=item_highlighted_code, is_displayed=is_displayed, ) user_url = settings.SPC['short_URL_root'] + 'user/' + str(user.id) if commit: # Save the submission, then the revision. If we are editing a # previous submission, then do not save the submission # (because the Submission object has fields that will never # change once it has been first created). if new_submission: # Sets the primary key sub.save() rev.entry_id = sub.id if not is_displayed: rev.validation_hash = create_validation_code(rev) rev.save() # Storage location: if we do save files it will be here datenow = datetime.datetime.now() year, month = datenow.strftime('%Y'), datenow.strftime('%m') repo_path = os.path.join(year, month, '%06d'% sub.id) full_repo_path = os.path.join(settings.SPC['storage_dir'], repo_path) if sub.sub_type == 'package': # Save the uploaded file to the server. At this point we are sure # it's a valid ZIP file, has no malicious filenames, and can be # unpacked to the hard drive. See validation in ``forms.py``. if os.path.exists(full_repo_path) and sub.fileset.repo_path == \ repo_path: # Make a temporary directory and copy the existing package # repository to that location temp_dir = tempfile.mkdtemp(prefix='tmp_spc_') src = os.path.join(full_repo_path, '.' + \ settings.SPC['revisioning_backend']) shutil.move(src, temp_dir) shutil.rmtree(full_repo_path, ignore_errors=True) else: temp_dir = None # Create/ensure destination directory exists ensuredir(full_repo_path) # Copy ZIP file zip_file = request.FILES['package_file'] dst = os.path.join(full_repo_path, zip_file.name) src = os.path.join(settings.SPC['ZIP_staging'], zip_file.name) shutil.copyfile(src, dst) # os.remove(src) Keep the original ZIP file, for now # Remove the entry from the database zip_hash = request.POST.get('package_hash', '') zip_objs = models.ZipFile.objects.filter(zip_hash=zip_hash) if zip_objs: zip_objs[0].delete() # Unzip file and commit contents to the repo zip_f = zipfile.ZipFile(dst, 'r') zip_f.extractall(full_repo_path) zip_f.close() os.remove(dst) # but delete the copy # Delete common RCS directories that might have been in the ZIP for path, dirs, files in os.walk(full_repo_path): if os.path.split(path)[1] in settings.SPC['common_rcs_dirs']: shutil.rmtree(path, ignore_errors=True) if temp_dir: src = os.path.join(temp_dir, '.' + \ settings.SPC['revisioning_backend']) dst = os.path.join(full_repo_path , '.' + \ settings.SPC['revisioning_backend']) try: os.rename(src, dst) except os.error, e: # For cases when /tmp is on a different filesystem # (usually production servers) import errno if e.errno == errno.EXDEV: shutil.copytree(src, dst, symlinks=True) shutil.rmtree(src) else: raise shutil.rmtree(temp_dir, ignore_errors=True) repo = sub.fileset.get_repo() else: # Create the repo sub.fileset = FileSet.objects.create(repo_path=repo_path) repo = sub.fileset.create_empty() sub.save() # Then add all files from the ZIP file to the repo. Add directories # at a time rather than file-by-file. for path, dirs, files in os.walk(full_repo_path): if os.path.split(path)[1] == '.' + \ settings.SPC['revisioning_backend']: for entry in dirs[:]: dirs.remove(entry) continue all_files = [] for name in files: all_files.append(os.path.join(path, name)) if all_files: repo.add(patterns=all_files, ignore_errors=True) # Add "DESCRIPTION.txt" descrip_name = os.path.join(full_repo_path, 'DESCRIPTION.txt') descrip_file = file(descrip_name, 'w') descrip_file.write(rev.description) descrip_file.close() sub.fileset.add_file(descrip_name, user=user_url, commit_msg=('Added/updated files from web-uploaded ' 'ZIP file. Added DESCRIPTION.txt also.')) if sub.sub_type == 'snippet': fname = rev.slug.replace('-', '_') + '.py' if new_submission: # Create a new repository for the files sub.fileset = FileSet.objects.create(repo_path=repo_path) sub.save() commit_msg = ('Add "%s" to the repo ' 'based on the web submission by user "%s"') %\ (fname, user_url) else: commit_msg = ('Update of file(s) in the repo ' 'based on the web submission by user "%s"') %\ (user_url) sub.fileset.add_file_from_string(fname, request.POST['snippet_code'], user=user_url, commit_msg=commit_msg) if sub.sub_type in ['snippet', 'package']: license_file = settings.SPC['license_filename'] license_text = get_license_text(rev) sub.fileset.add_file_from_string(license_file, license_text, user="******", commit_msg="SPC: added/updated license file" ) rev.hash_id = sub.fileset.get_hash() # Once you have the revision you can add tags through the intermediate # model instance (which tracks the user that added the tag and when). for tag in tag_list: tag_intermediate = models.TagCreation(created_by=user, revision=rev, tag=tag) tag_intermediate.save() logger.debug('User "%s" added tag "%s" to rev.id=%d' % ( user.profile.slug, str(tag), rev.id)) # Update the search index so that the tags are included in the search rev.save() # log the new submission and revision logger.info('New %s: %s [id=%d] and revision id=%d' % ( sub.sub_type, item.cleaned_data['title'], sub.id, rev.id))