def handle(self, *args, **kwargs):
for note in Note.objects.filter(fp_file='').iterator():
if note.fp_file.name:
print "Skipping {0}".format(str(note))
continue
if note.static_html:
# grab the html from inside the note and process it
html_resp = requests.get('http:{0}{1}'.format(settings.S3_URL, note.get_relative_s3_path()))
if html_resp.status_code is not 200:
print html_resp.text
continue
html = html_resp.text
elif note.html:
html = note.html
else:
print "Couldn't find any HTML for {0}".format(str(note))
continue
fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
fp_policy = encode_fp_policy(fp_policy_json)
fp_signature = sign_fp_policy(fp_policy)
upload_resp = requests.post('https://www.filepicker.io/api/store/S3',
params={'key': FILEPICKER_API_KEY,
'policy': fp_policy,
'signature': fp_signature,
'filename': slugify(note.name)},
headers={'Content-Type': 'text/html; charset=utf-8'},
data=html.encode('utf-8'))
if upload_resp.status_code is not 200:
print upload_resp.text
continue
resp_json = json.loads(upload_resp.text)
url = resp_json['url']
note.fp_file = url
get_resp = requests.get(url,
params={'key': FILEPICKER_API_KEY,
'policy': fp_policy,
'signature': fp_signature})
if get_resp.status_code is not 200:
print "It looks like this note upload did not succeed"
print str(note)
continue
if get_resp.text != html:
print "The content at the new Filepicker URL does not match the original note contents!"
note.save()
print "Successfully fixed {0}".format(str(note))
EMAIL_HIT_QUALIFICATION = Qualifications(requirements=[KEYWORDS_HIT_PERCENT_APPROVED_REQUIREMENT])
COURSE_SPAM_QID = 'course_spam'
COURSE_NAME_QID = 'course_name'
INSTRUCTOR_NAMES_QID = 'instructor_names'
SCHOOL_NAME_QID = 'school_name'
DEPARTMENT_NAME_QID = 'department_name'
NOTE_CATEGORY_QID_TEMPLATE = 'note_category_'
NOTE_TITLE_QID_TEMPLATE = 'note_title_'
NOTE_CATEGORIES_FOR_MTURK = [(c[1], c[0]) for c in Document.NOTE_CATEGORIES]
FP_POLICY_JSON_READ_WRITE = '{{"expiry": {0}, "call": ["store","read","stat"]}}'
FP_POLICY_JSON_READ_WRITE = FP_POLICY_JSON_READ_WRITE.format(int(time.time() + 31536000))
FP_POLICY_READ_WRITE = encode_fp_policy(FP_POLICY_JSON_READ_WRITE)
FP_SIGNATURE_READ_WRITE = sign_fp_policy(FP_POLICY_READ_WRITE)
FP_POLICY_JSON_READ = '{{"expiry": {0}, "call": ["read","stat"]}}'
FP_POLICY_JSON_READ = FP_POLICY_JSON_READ.format(int(time.time() + 31536000))
FP_POLICY_READ = encode_fp_policy(FP_POLICY_JSON_READ)
FP_SIGNATURE_READ = sign_fp_policy(FP_POLICY_READ)
CONTENT_DISPOSITION_REGEX = r'filename="(?P<filename>.+)"'
@task(name='check_notes_mailbox')
def check_notes_mailbox():
try:
MAILBOX_USER = os.environ['NOTES_MAILBOX_USERNAME']
MAILBOX_PASSWORD = os.environ['NOTES_MAILBOX_PASSWORD']
def handle(self, *args, **kwargs):
for note in Note.objects.filter(fp_file='').iterator():
if note.fp_file.name:
print "Skipping {0}".format(str(note))
continue
if note.static_html:
# grab the html from inside the note and process it
html_resp = requests.get('http:{0}{1}'.format(
settings.S3_URL, note.get_relative_s3_path()))
if html_resp.status_code is not 200:
print html_resp.text
continue
html = html_resp.text
elif note.html:
html = note.html
else:
print "Couldn't find any HTML for {0}".format(str(note))
continue
fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
fp_policy = encode_fp_policy(fp_policy_json)
fp_signature = sign_fp_policy(fp_policy)
upload_resp = requests.post(
'https://www.filepicker.io/api/store/S3',
params={
'key': FILEPICKER_API_KEY,
'policy': fp_policy,
'signature': fp_signature,
'filename': slugify(note.name)
},
headers={'Content-Type': 'text/html; charset=utf-8'},
data=html.encode('utf-8'))
if upload_resp.status_code is not 200:
print upload_resp.text
continue
resp_json = json.loads(upload_resp.text)
url = resp_json['url']
note.fp_file = url
get_resp = requests.get(url,
params={
'key': FILEPICKER_API_KEY,
'policy': fp_policy,
'signature': fp_signature
})
if get_resp.status_code is not 200:
print "It looks like this note upload did not succeed"
print str(note)
continue
if get_resp.text != html:
print "The content at the new Filepicker URL does not match the original note contents!"
note.save()
print "Successfully fixed {0}".format(str(note))
class Document(models.Model):
"""
An Abstract Base Class representing a document intended to be subclassed.
"""
course = models.ForeignKey(Course)
tags = TaggableManager(blank=True)
name = models.CharField(max_length=255, blank=True, null=True)
slug = models.SlugField(max_length=255, unique=True)
LECTURE_NOTES = 'LECTURE_NOTES'
STUDY_GUIDE = 'STUDY_GUIDE'
SYLLABUS = 'SYLLABUS'
ASSIGNMENT = 'ASSIGNMENT'
OTHER = 'OTHER'
NOTE_CATEGORIES = (
(LECTURE_NOTES, 'Lecture Notes'),
(STUDY_GUIDE, 'Study Guide'),
(SYLLABUS, 'Syllabus'),
(ASSIGNMENT, 'Assignment'),
(OTHER, 'Other'),
)
EDITABLE_CATEGORIES = (LECTURE_NOTES, )
category = models.CharField(max_length=50,
choices=NOTE_CATEGORIES,
blank=True,
null=True)
# license if different from default
license = models.ForeignKey(License, blank=True, null=True)
# provide an upstream file link
upstream_link = models.URLField(max_length=1024,
blank=True,
null=True,
unique=True)
# metadata relevant to the Upload process
user = models.ForeignKey(User, blank=True, null=True, on_delete=SET_NULL)
ip = models.GenericIPAddressField(blank=True,
null=True,
help_text=u"IP address of the uploader")
uploaded_at = models.DateTimeField(null=True,
default=datetime.datetime.utcnow)
# if True, NEVER show this file
# WARNING: This may throw an error on migration
is_hidden = models.BooleanField(default=False)
###
# Everything Filepicker, now in one small area
# Allow pick (choose files), store (upload to S3), read (from FP repo),
# stat (status of FP repo files) for 1 year (current time + 365 * 24 * 3600
# seconds). Generated one time, at class definition upon import. So the
# server will need to be rebooted at least one time each year or this will
# go stale.
fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
fp_policy = encode_fp_policy(fp_policy_json)
fp_signature = sign_fp_policy(fp_policy)
# Hack because mimetypes conflict with extensions, but there is no way to
# disable mimetypes.
# https://github.com/Ink/django-filepicker/issues/22
django_filepicker.forms.FPFieldMixin.default_mimetypes = ''
# Now let django-filepicker do the heavy lifting. Sort of. Look at all those
# parameters!
fp_file = django_filepicker.models.FPFileField(
# FPFileField settings
apikey=FILEPICKER_API_KEY,
services=
'COMPUTER,DROPBOX,URL,GOOGLE_DRIVE,EVERNOTE,GMAIL,BOX,FACEBOOK,FLICKR,PICASA,IMAGE_SEARCH,WEBCAM,FTP',
additional_params={
'data-fp-multiple': 'true',
'data-fp-folders': 'true',
'data-fp-button-class': 'inline-button important add-note-btn',
'data-fp-button-text': 'Add Notes',
'data-fp-extensions':
'.pdf,.doc,.docx,.txt,.html,.rtf,.odt,.png,.jpg,.jpeg,.ppt,.pptx',
'data-fp-store-location': 'S3',
'data-fp-policy': fp_policy,
'data-fp-signature': fp_signature,
'type': 'filepicker',
'onchange': "got_file(event)",
},
# FileField settings
null=True,
blank=True,
upload_to='nil', # field ignored because S3, but required.
verbose_name='', # prevent a label from showing up
)
mimetype = models.CharField(max_length=255, blank=True, null=True)
class Meta:
abstract = True
ordering = ['-uploaded_at']
def _generate_unique_slug(self):
""" generate a unique slug based on name and uploaded_at """
_slug = slugify(unicode(self.name))
klass = self.__class__
collision = klass.objects.filter(slug=_slug)
if collision:
_slug = u"{0}-{1}-{2}-{3}".format(_slug, self.uploaded_at.month,
self.uploaded_at.day,
self.uploaded_at.microsecond)
self.slug = _slug
def _get_fpf(self):
"""
Memoized FilepickerFile getter. Returns FilepickerFile.
"""
if not hasattr(self, 'cached_fpf'):
# Fetch additional_params containing signature, etc
aps = self.fp_file.field.additional_params
self.cached_fpf = django_filepicker.utils.FilepickerFile(
self.fp_file.name, aps)
return self.cached_fpf
def get_fp_url(self):
"""
Returns the Filepicker URL for reading the upstream document.
"""
fpf = self._get_fpf()
# Return proper URL for reading
return fpf.get_url()
def get_file(self):
"""
Downloads the file from filepicker.io and returns a Django File wrapper
object.
"""
# Fetch FilepickerFile
fpf = self._get_fpf()
# Return Django File
return fpf.get_file()
def save(self, *args, **kwargs):
if self.name and not self.slug:
self._generate_unique_slug()
super(Document, self).save(*args, **kwargs)
COURSE_SPAM_QID = 'course_spam'
COURSE_NAME_QID = 'course_name'
INSTRUCTOR_NAMES_QID = 'instructor_names'
SCHOOL_NAME_QID = 'school_name'
DEPARTMENT_NAME_QID = 'department_name'
NOTE_CATEGORY_QID_TEMPLATE = 'note_category_'
NOTE_TITLE_QID_TEMPLATE = 'note_title_'
NOTE_CATEGORIES_FOR_MTURK = [(c[1], c[0]) for c in Document.NOTE_CATEGORIES]
FP_POLICY_JSON_READ_WRITE = '{{"expiry": {0}, "call": ["store","read","stat"]}}'
FP_POLICY_JSON_READ_WRITE = FP_POLICY_JSON_READ_WRITE.format(
int(time.time() + 31536000))
FP_POLICY_READ_WRITE = encode_fp_policy(FP_POLICY_JSON_READ_WRITE)
FP_SIGNATURE_READ_WRITE = sign_fp_policy(FP_POLICY_READ_WRITE)
FP_POLICY_JSON_READ = '{{"expiry": {0}, "call": ["read","stat"]}}'
FP_POLICY_JSON_READ = FP_POLICY_JSON_READ.format(int(time.time() + 31536000))
FP_POLICY_READ = encode_fp_policy(FP_POLICY_JSON_READ)
FP_SIGNATURE_READ = sign_fp_policy(FP_POLICY_READ)
CONTENT_DISPOSITION_REGEX = r'filename="(?P<filename>.+)"'
@task(name='check_notes_mailbox')
def check_notes_mailbox():
MTURK_HOST = run_mturk('get_extract_keywords_results')
if not MTURK_HOST:
return