def _create_submission_pages(submission):
"""
Creates submission pages for the given PDF. Employs Evangelist to perform
PDF -> JPEG conversion.
"""
random_prefix = utils.generate_random_string(50)
jpeg_path = 'homework-pages/%s%%d.jpeg' % random_prefix
small_jpeg_path = 'homework-pages/%s%%d-small.jpeg' % random_prefix
large_jpeg_path = 'homework-pages/%s%%d-large.jpeg' % random_prefix
pages_to_save = []
print 'Preparing submission pages...'
for page_number in xrange(1, submission.page_count + 1):
# paths to normal, small, and large JPEGs; these haven't been uploaded yet
jpeg_path_for_page = jpeg_path % page_number
jpeg_field = file_fields.ImageFieldFile(instance=None,
field=file_fields.FileField(),
name=jpeg_path_for_page)
small_jpeg_path_for_page = small_jpeg_path % page_number
small_jpeg_field = file_fields.ImageFieldFile(
instance=None,
field=file_fields.FileField(),
name=small_jpeg_path_for_page)
large_jpeg_path_for_page = large_jpeg_path % page_number
large_jpeg_field = file_fields.ImageFieldFile(
instance=None,
field=file_fields.FileField(),
name=large_jpeg_path_for_page)
# prepare all submission pages
submission_page = models.SubmissionPage(
submission=submission,
page_number=page_number,
page_jpeg=jpeg_field,
is_blank=False,
page_jpeg_small=small_jpeg_field,
page_jpeg_large=large_jpeg_field)
pages_to_save.append(submission_page)
print 'Making request to Evangelist...'
response_text = evangelist.convert_pdf_to_jpegs(submission.pdf.name,
jpeg_path, small_jpeg_path,
large_jpeg_path)
print 'Got Evangelist response text: %s' % response_text
# finalize all submission pages
for submission_page in pages_to_save:
submission_page.save()
print 'Finalized submission pages!'
def test_django_image_files(self):
from django.db.models.fields import files
field = files.FileField()
f = SourceFile(files.FieldFile(field=field, instance=None, name=self.FILE_PATH))
self.assertEqual(f.file, self.FILE_PATH)
f = SourceFile(files.ImageFieldFile(field=field, instance=None, name=self.FILE_PATH))
self.assertEqual(f.file, self.FILE_PATH)
def test_get_thumbnail_templatetag_with_FileField_as_source(
self, mock_get_thumbnail):
field = files.FileField()
_file = files.FieldFile(field=field, instance=None, name='image.jpg')
Template(
'{% load thumbnails %}'
'{% get_thumbnail image "200x200" as im %}'
'{{im.url}}', ).render(Context({'image': _file}))
mock_get_thumbnail.assert_called_once_with(_file, '200x200')
class File(models.Model):
created = models.DateTimeField(auto_now_add=True)
owner = models.ForeignKey(User, models.CASCADE, db_index=True)
project = models.ForeignKey(Project, models.CASCADE, null=True,
related_name='files')
name = models.CharField(max_length=256)
description = models.TextField()
md5hash = models.CharField(max_length=32, db_index=True,
validators=[MinLengthValidator(32)])
file = files.FileField(upload_to="tasks", null=True,
validators=[idb_validator])
def __unicode__(self):
return "File {}".format(self.name)
__str__ = __unicode__
def _create_and_upload_split_pages(split, num_pages, num_pages_per_exam,
is_single):
"""
For each page in the pdf, create a `SplitPage` and associated a JPEG with it.
Runs the PDF -> JPEG converter worker for all pages, uploading the
JPEGs to S3.
"""
NUM_PAGES_PER_WORKER = 500
num_workers = (num_pages - 1) / NUM_PAGES_PER_WORKER + 1
for worker in range(num_workers):
print 'Spawning worker %d' % worker
offset = worker * NUM_PAGES_PER_WORKER
pages = range(offset, min(num_pages, offset + NUM_PAGES_PER_WORKER))
jpeg_prefixes = map(
lambda student: 'split-pages/%s' % utils.generate_random_string(
40), pages)
for page in pages:
# We make a guess for `begins_submission` by assuming all the exams
# uploaded have the correct number of pages
split_page = models.SplitPage(
split=split,
page_number=page + 1,
begins_submission=(page % num_pages_per_exam == 0),
is_single=is_single)
jpeg_name = '%s.jpeg' % jpeg_prefixes[page - offset]
jpeg_field = file_fields.ImageFieldFile(
instance=None, field=file_fields.FileField(), name=jpeg_name)
small_jpeg_name = '%s-small.jpeg' % jpeg_prefixes[page - offset]
small_jpeg_field = file_fields.ImageFieldFile(
instance=None,
field=file_fields.FileField(),
name=small_jpeg_name)
large_jpeg_name = '%s-large.jpeg' % jpeg_prefixes[page - offset]
large_jpeg_field = file_fields.ImageFieldFile(
instance=None,
field=file_fields.FileField(),
name=large_jpeg_name)
split_page.page_jpeg = jpeg_field
split_page.page_jpeg_small = small_jpeg_field
split_page.page_jpeg_large = large_jpeg_field
split_page.save()
dp = dispatcher.Dispatcher()
# spawn thread to dispatch converter worker
payload = {
's3': {
'token': settings.AWS_S3_ACCESS_KEY_ID,
'secret': settings.AWS_S3_SECRET_ACCESS_KEY,
'bucket': settings.AWS_STORAGE_BUCKET_NAME,
},
'webhook_data': {
'split_id': split.id,
'secret': split.secret
},
'webhook_url': settings.SITE_URL + 'update-split-page-state/',
'pdf_path': split.pdf.name,
'jpeg_prefixes': jpeg_prefixes,
'page_start': pages[0],
'page_end': pages[-1]
}
instance_options = {'instance_type': 'm3.medium'}
dispatch_worker.delay(dp, 'converter', payload, instance_options)