Ejemplo n.º 1
0
    def handle(self, *args, **kwargs):
        for note in Note.objects.filter(fp_file='').iterator():
            if note.fp_file.name:
                print "Skipping {0}".format(str(note))
                continue

            if note.static_html:
                # grab the html from inside the note and process it
                html_resp = requests.get('http:{0}{1}'.format(settings.S3_URL, note.get_relative_s3_path()))
                if html_resp.status_code is not 200:
                    print html_resp.text
                    continue
                html = html_resp.text
            elif note.html:
                html = note.html
            else:
                print "Couldn't find any HTML for {0}".format(str(note))
                continue

            fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
            fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
            fp_policy      = encode_fp_policy(fp_policy_json)
            fp_signature   = sign_fp_policy(fp_policy)

            upload_resp = requests.post('https://www.filepicker.io/api/store/S3',
                          params={'key': FILEPICKER_API_KEY,
                                  'policy': fp_policy,
                                  'signature': fp_signature,
                                  'filename': slugify(note.name)},
                          headers={'Content-Type': 'text/html; charset=utf-8'},
                          data=html.encode('utf-8'))

            if upload_resp.status_code is not 200:
                print upload_resp.text
                continue

            resp_json = json.loads(upload_resp.text)
            url = resp_json['url']
            note.fp_file = url

            get_resp = requests.get(url,
                                    params={'key': FILEPICKER_API_KEY,
                                            'policy': fp_policy,
                                            'signature': fp_signature})

            if get_resp.status_code is not 200:
                print "It looks like this note upload did not succeed"
                print str(note)
                continue

            if get_resp.text != html:
                print "The content at the new Filepicker URL does not match the original note contents!"

            note.save()

            print "Successfully fixed {0}".format(str(note))
Ejemplo n.º 2
0
EMAIL_HIT_QUALIFICATION = Qualifications(requirements=[KEYWORDS_HIT_PERCENT_APPROVED_REQUIREMENT])

COURSE_SPAM_QID = 'course_spam'
COURSE_NAME_QID = 'course_name'
INSTRUCTOR_NAMES_QID = 'instructor_names'
SCHOOL_NAME_QID = 'school_name'
DEPARTMENT_NAME_QID = 'department_name'

NOTE_CATEGORY_QID_TEMPLATE = 'note_category_'
NOTE_TITLE_QID_TEMPLATE = 'note_title_'

NOTE_CATEGORIES_FOR_MTURK = [(c[1], c[0]) for c in Document.NOTE_CATEGORIES]

FP_POLICY_JSON_READ_WRITE = '{{"expiry": {0}, "call": ["store","read","stat"]}}'
FP_POLICY_JSON_READ_WRITE = FP_POLICY_JSON_READ_WRITE.format(int(time.time() + 31536000))
FP_POLICY_READ_WRITE      = encode_fp_policy(FP_POLICY_JSON_READ_WRITE)
FP_SIGNATURE_READ_WRITE   = sign_fp_policy(FP_POLICY_READ_WRITE)

FP_POLICY_JSON_READ = '{{"expiry": {0}, "call": ["read","stat"]}}'
FP_POLICY_JSON_READ = FP_POLICY_JSON_READ.format(int(time.time() + 31536000))
FP_POLICY_READ      = encode_fp_policy(FP_POLICY_JSON_READ)
FP_SIGNATURE_READ   = sign_fp_policy(FP_POLICY_READ)

CONTENT_DISPOSITION_REGEX = r'filename="(?P<filename>.+)"'


@task(name='check_notes_mailbox')
def check_notes_mailbox():
    try:
        MAILBOX_USER = os.environ['NOTES_MAILBOX_USERNAME']
        MAILBOX_PASSWORD = os.environ['NOTES_MAILBOX_PASSWORD']
Ejemplo n.º 3
0
    def handle(self, *args, **kwargs):
        for note in Note.objects.filter(fp_file='').iterator():
            if note.fp_file.name:
                print "Skipping {0}".format(str(note))
                continue

            if note.static_html:
                # grab the html from inside the note and process it
                html_resp = requests.get('http:{0}{1}'.format(
                    settings.S3_URL, note.get_relative_s3_path()))
                if html_resp.status_code is not 200:
                    print html_resp.text
                    continue
                html = html_resp.text
            elif note.html:
                html = note.html
            else:
                print "Couldn't find any HTML for {0}".format(str(note))
                continue

            fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
            fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
            fp_policy = encode_fp_policy(fp_policy_json)
            fp_signature = sign_fp_policy(fp_policy)

            upload_resp = requests.post(
                'https://www.filepicker.io/api/store/S3',
                params={
                    'key': FILEPICKER_API_KEY,
                    'policy': fp_policy,
                    'signature': fp_signature,
                    'filename': slugify(note.name)
                },
                headers={'Content-Type': 'text/html; charset=utf-8'},
                data=html.encode('utf-8'))

            if upload_resp.status_code is not 200:
                print upload_resp.text
                continue

            resp_json = json.loads(upload_resp.text)
            url = resp_json['url']
            note.fp_file = url

            get_resp = requests.get(url,
                                    params={
                                        'key': FILEPICKER_API_KEY,
                                        'policy': fp_policy,
                                        'signature': fp_signature
                                    })

            if get_resp.status_code is not 200:
                print "It looks like this note upload did not succeed"
                print str(note)
                continue

            if get_resp.text != html:
                print "The content at the new Filepicker URL does not match the original note contents!"

            note.save()

            print "Successfully fixed {0}".format(str(note))
Ejemplo n.º 4
0
class Document(models.Model):
    """
    An Abstract Base Class representing a document intended to be subclassed.
    """
    course = models.ForeignKey(Course)
    tags = TaggableManager(blank=True)
    name = models.CharField(max_length=255, blank=True, null=True)
    slug = models.SlugField(max_length=255, unique=True)

    LECTURE_NOTES = 'LECTURE_NOTES'
    STUDY_GUIDE = 'STUDY_GUIDE'
    SYLLABUS = 'SYLLABUS'
    ASSIGNMENT = 'ASSIGNMENT'
    OTHER = 'OTHER'
    NOTE_CATEGORIES = (
        (LECTURE_NOTES, 'Lecture Notes'),
        (STUDY_GUIDE, 'Study Guide'),
        (SYLLABUS, 'Syllabus'),
        (ASSIGNMENT, 'Assignment'),
        (OTHER, 'Other'),
    )
    EDITABLE_CATEGORIES = (LECTURE_NOTES, )

    category = models.CharField(max_length=50,
                                choices=NOTE_CATEGORIES,
                                blank=True,
                                null=True)

    # license if different from default
    license = models.ForeignKey(License, blank=True, null=True)

    # provide an upstream file link
    upstream_link = models.URLField(max_length=1024,
                                    blank=True,
                                    null=True,
                                    unique=True)

    # metadata relevant to the Upload process
    user = models.ForeignKey(User, blank=True, null=True, on_delete=SET_NULL)
    ip = models.GenericIPAddressField(blank=True,
                                      null=True,
                                      help_text=u"IP address of the uploader")
    uploaded_at = models.DateTimeField(null=True,
                                       default=datetime.datetime.utcnow)

    # if True, NEVER show this file
    # WARNING: This may throw an error on migration
    is_hidden = models.BooleanField(default=False)

    ###
    # Everything Filepicker, now in one small area

    # Allow pick (choose files), store (upload to S3), read (from FP repo),
    # stat (status of FP repo files) for 1 year (current time + 365 * 24 * 3600
    # seconds). Generated one time, at class definition upon import. So the
    # server will need to be rebooted at least one time each year or this will
    # go stale.
    fp_policy_json = '{{"expiry": {0}, "call": ["pick","store","read","stat"]}}'
    fp_policy_json = fp_policy_json.format(int(time.time() + 31536000))
    fp_policy = encode_fp_policy(fp_policy_json)
    fp_signature = sign_fp_policy(fp_policy)

    # Hack because mimetypes conflict with extensions, but there is no way to
    # disable mimetypes.
    # https://github.com/Ink/django-filepicker/issues/22
    django_filepicker.forms.FPFieldMixin.default_mimetypes = ''
    # Now let django-filepicker do the heavy lifting. Sort of. Look at all those
    # parameters!
    fp_file = django_filepicker.models.FPFileField(
        # FPFileField settings
        apikey=FILEPICKER_API_KEY,
        services=
        'COMPUTER,DROPBOX,URL,GOOGLE_DRIVE,EVERNOTE,GMAIL,BOX,FACEBOOK,FLICKR,PICASA,IMAGE_SEARCH,WEBCAM,FTP',
        additional_params={
            'data-fp-multiple': 'true',
            'data-fp-folders': 'true',
            'data-fp-button-class': 'inline-button important add-note-btn',
            'data-fp-button-text': 'Add Notes',
            'data-fp-extensions':
            '.pdf,.doc,.docx,.txt,.html,.rtf,.odt,.png,.jpg,.jpeg,.ppt,.pptx',
            'data-fp-store-location': 'S3',
            'data-fp-policy': fp_policy,
            'data-fp-signature': fp_signature,
            'type': 'filepicker',
            'onchange': "got_file(event)",
        },
        # FileField settings
        null=True,
        blank=True,
        upload_to='nil',  # field ignored because S3, but required.
        verbose_name='',  # prevent a label from showing up
    )
    mimetype = models.CharField(max_length=255, blank=True, null=True)

    class Meta:
        abstract = True
        ordering = ['-uploaded_at']

    def _generate_unique_slug(self):
        """ generate a unique slug based on name and uploaded_at  """
        _slug = slugify(unicode(self.name))
        klass = self.__class__
        collision = klass.objects.filter(slug=_slug)
        if collision:
            _slug = u"{0}-{1}-{2}-{3}".format(_slug, self.uploaded_at.month,
                                              self.uploaded_at.day,
                                              self.uploaded_at.microsecond)
        self.slug = _slug

    def _get_fpf(self):
        """
        Memoized FilepickerFile getter. Returns FilepickerFile.
        """
        if not hasattr(self, 'cached_fpf'):
            # Fetch additional_params containing signature, etc
            aps = self.fp_file.field.additional_params
            self.cached_fpf = django_filepicker.utils.FilepickerFile(
                self.fp_file.name, aps)
        return self.cached_fpf

    def get_fp_url(self):
        """
        Returns the Filepicker URL for reading the upstream document.
        """
        fpf = self._get_fpf()
        # Return proper URL for reading
        return fpf.get_url()

    def get_file(self):
        """
        Downloads the file from filepicker.io and returns a Django File wrapper
        object.
        """
        # Fetch FilepickerFile
        fpf = self._get_fpf()
        # Return Django File
        return fpf.get_file()

    def save(self, *args, **kwargs):
        if self.name and not self.slug:
            self._generate_unique_slug()
        super(Document, self).save(*args, **kwargs)
Ejemplo n.º 5
0
COURSE_SPAM_QID = 'course_spam'
COURSE_NAME_QID = 'course_name'
INSTRUCTOR_NAMES_QID = 'instructor_names'
SCHOOL_NAME_QID = 'school_name'
DEPARTMENT_NAME_QID = 'department_name'

NOTE_CATEGORY_QID_TEMPLATE = 'note_category_'
NOTE_TITLE_QID_TEMPLATE = 'note_title_'

NOTE_CATEGORIES_FOR_MTURK = [(c[1], c[0]) for c in Document.NOTE_CATEGORIES]

FP_POLICY_JSON_READ_WRITE = '{{"expiry": {0}, "call": ["store","read","stat"]}}'
FP_POLICY_JSON_READ_WRITE = FP_POLICY_JSON_READ_WRITE.format(
    int(time.time() + 31536000))
FP_POLICY_READ_WRITE = encode_fp_policy(FP_POLICY_JSON_READ_WRITE)
FP_SIGNATURE_READ_WRITE = sign_fp_policy(FP_POLICY_READ_WRITE)

FP_POLICY_JSON_READ = '{{"expiry": {0}, "call": ["read","stat"]}}'
FP_POLICY_JSON_READ = FP_POLICY_JSON_READ.format(int(time.time() + 31536000))
FP_POLICY_READ = encode_fp_policy(FP_POLICY_JSON_READ)
FP_SIGNATURE_READ = sign_fp_policy(FP_POLICY_READ)

CONTENT_DISPOSITION_REGEX = r'filename="(?P<filename>.+)"'


@task(name='check_notes_mailbox')
def check_notes_mailbox():
    MTURK_HOST = run_mturk('get_extract_keywords_results')
    if not MTURK_HOST:
        return