Example #1
0
    def post(self):

        form = AddItemForm()
        item = Item()

        if form.validate_on_submit():
            ar_title = Titles()
            fr_title = Titles()
            en_title = Titles()

            ar_title.title = form.ar_title.data.strip()
            ar_title.lang = 'ar'

            fr_title.title = form.fr_title.data.strip()
            fr_title.lang = 'fr'

            en_title.title = form.en_title.data.strip()
            en_title.lang = 'en'

            item.titles.append(ar_title)
            item.titles.append(fr_title)
            item.titles.append(en_title)

            item.description = form.description.data

            item.submitter = User.objects.get(id=current_user.id)

        else:
            flash('upload unsuccessful', 'error')
            return render_template('items/add_item.html', form=form)

        uploaded_files = request.files.getlist("files")
        thumbnail = request.files['thumbnail']

        thumbnail_name = secure_filename(thumbnail.filename)

        if thumbnail and allowed_thumbnails(thumbnail_name):
            ext = thumbnail.mimetype.split('/')[-1]
            # use the 'thumbnail' name for all thumbnails
            filename = '.'.join(["thumbnail", ext])
            item.thumbnail.put(thumbnail.stream,
                               content_type=thumbnail.mimetype,
                               filename=filename)

        for file in uploaded_files:
            # Make the filename safe, remove unsupported chars
            filename = secure_filename(file.filename)
            # Check if the file is one of the allowed types/extensions
            if file and allowed_file(filename):
                # put the file in the ListField.
                # see https://gist.github.com/tfausak/1299339
                file_ = GridFSProxy()
                file_.put(file.stream,
                          content_type=file.mimetype,
                          filename=filename)
                item.files.append(file_)
        # Save the thing
        item.save()
        flash('upload successful')
        return render_template('items/add_item.html', form=form)
Example #2
0
    def put(self, obj, name, attributes=None):
        """
        Store a NotebookNode

        :param obj: the NotebookNode to store
        :param name: the name of the notebook
        """
        if not name.endswith('.ipynb'):
            name += '.ipynb'
        sbuf = StringIO()
        bbuf = BytesIO()
        # nbwrite expects string, fs.put expects bytes
        nbwrite(obj, sbuf, version=4)
        sbuf.seek(0)
        bbuf.write(sbuf.getvalue().encode('utf8'))
        bbuf.seek(0)
        # see if we have a file already, if so replace the gridfile
        meta = self.store.metadata(name)
        if not meta:
            filename = uuid4().hex
            fileid = self._fs.put(bbuf, filename=filename)
            meta = self.store._make_metadata(
                name=name,
                prefix=self.store.prefix,
                bucket=self.store.bucket,
                kind=self.kind,
                attributes=attributes,
                gridfile=GridFSProxy(grid_id=fileid))
        else:
            meta.gridfile.replace(bbuf)
        return meta.save()
Example #3
0
    def put(self, obj, name, attributes=None):
        """
        Store a NotebookNode

        :param obj: the NotebookNode to store
        :param name: the name of the notebook
        """
        if not name.endswith('.ipynb'):
            name += '.ipynb'
        sbuf = StringIO()
        bbuf = BytesIO()
        # nbwrite expects string, fs.put expects bytes
        nbwrite(obj, sbuf, version=4)
        sbuf.seek(0)
        bbuf.write(sbuf.getvalue().encode('utf8'))
        bbuf.seek(0)
        # see if we have a file already, if so replace the gridfile
        meta = self.store.metadata(name)
        if not meta:
            filename = uuid4().hex
            fileid = self._fs.put(bbuf, filename=filename)
            meta = self.store._make_metadata(
                name=name,
                prefix=self.store.prefix,
                bucket=self.store.bucket,
                kind=self.kind,
                attributes=attributes,
                gridfile=GridFSProxy(grid_id=fileid))
            meta = meta.save()
        else:
            filename = uuid4().hex
            meta.gridfile.delete()
            fileid = self._fs.put(bbuf, filename=filename)
            meta.gridfile = GridFSProxy(grid_id=fileid)
            meta = meta.save()
        # set config
        nb_config = self.get_notebook_config(name)
        meta_config = meta.attributes.get('config', {})
        if nb_config:
            meta_config.update(dict(**nb_config))
            meta.attributes['config'] = meta_config
        meta = meta.save()
        if not name.startswith('results') and ('run-at' in meta_config):
            meta = self.schedule(name)
        return meta
Example #4
0
def upload(space='main', doc_id=None):
    print 'enter request'
    if doc_id is None:
        redirect(url_for('.dashboard'))
    else:
        doc = BaseDocument.objects(id=doc_id).first()
        # check acl for document
        acl.apply('write', doc.acl, _('document'))

    print request.files

    file = request.files['upload']
    file_ = GridFSProxy()
    file_.put(file.read(), content_type=file.content_type, filename=file.filename)
    doc.attachments.append(file_)
    doc.save()

    flash(_('File uploaded successfully.'), 'info')
    return redirect(url_for('.view', space=space, doc_id=doc.id))
Example #5
0
File: base.py Project: 0r0i/omegaml
 def put_dataframe_as_hdf(self, obj, name, attributes=None):
     filename = self._get_obj_store_key(name, '.hdf')
     hdffname = self._package_dataframe2hdf(obj, filename)
     with open(hdffname, 'rb') as fhdf:
         fileid = self.fs.put(fhdf, filename=filename)
     return self._make_metadata(name=name,
                                prefix=self.prefix,
                                bucket=self.bucket,
                                kind=MDREGISTRY.PANDAS_HDF,
                                attributes=attributes,
                                gridfile=GridFSProxy(grid_id=fileid)).save()
Example #6
0
 def put_model(self, obj, name, attributes=None):
     """
     Packages a model using joblib and stores in GridFS
     """
     zipfname = self._package_model(obj, name)
     with open(zipfname, 'rb') as fzip:
         fileid = self.model_store.fs.put(
             fzip, filename=self.model_store._get_obj_store_key(name, 'omm'))
         gridfile = GridFSProxy(grid_id=fileid,
                                db_alias='omega',
                                collection_name=self.model_store.bucket)
     return self.model_store._make_metadata(
         name=name,
         prefix=self.model_store.prefix,
         bucket=self.model_store.bucket,
         kind=MDREGISTRY.SKLEARN_JOBLIB,
         attributes=attributes,
         gridfile=gridfile).save()
Example #7
0
def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
    """
    Constructs and returns a document instance from the bound ``form``'s
    ``cleaned_data``, but does not save the returned instance to the
    database.
    """
    cleaned_data = form.cleaned_data
    file_field_list = []
    
    # check wether object is instantiated
    if isinstance(instance, type):
        instance = instance()
        
    for f in instance._fields.values():
        if isinstance(f, ObjectIdField):
            continue
        if not f.name in cleaned_data:
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        # Defer saving file-type fields until after the other fields, so a
        # callable upload_to can use the values from other fields.
        if isinstance(f, FileField) or (isinstance(f, (MapField, ListField)) and isinstance(f.field, FileField)):
            file_field_list.append(f)
        else:
            setattr(instance, f.name, cleaned_data.get(f.name))

    for f in file_field_list:
        if isinstance(f, MapField):
            map_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for key, uploaded_file in uploads.items():
                if uploaded_file is None:
                    continue
                file_data = map_field.get(key, None)
                map_field[key] = _save_iterator_file(f, uploaded_file, file_data)
            setattr(instance, f.name, map_field)
        elif isinstance(f, ListField):
            list_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for i, uploaded_file in enumerate(uploads):
                if uploaded_file is None:
                    continue
                try:
                    file_data = list_field[i]
                except IndexError:
                    file_data = None
                file_obj = _save_iterator_file(f, uploaded_file, file_data)
                try:
                    list_field[i] = file_obj
                except IndexError:
                    list_field.append(file_obj)
            setattr(instance, f.name, list_field)
        else:
            field = getattr(instance, f.name)
            upload = cleaned_data[f.name]
            if upload is None:
                continue
            
            try:
                print "el tipo en field es %s" % type(field)
                
                upload.file.seek(0)
                # delete first to get the names right
                #if field.grid_id:
                #    field.delete()
                from mongoengine.fields import GridFSProxy, ImageGridFsProxy
                if type(field) == type(GridFSProxy()):
                    print "fichero"
                    if field.grid_id:
                        field = GridFSProxy()

                    import hashlib
                    try:
                        grid_id = _get_grid_id(hashlib.md5(upload.file.read()).hexdigest(), f.db_alias, f.collection_name)
                    except Exception, e:
                        print "Error: %s" % e

                    if grid_id == None:
                        upload.file.seek(0)
                        filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
                        field.put(upload, content_type=upload.content_type, filename=filename)
                    else:
                        field.grid_id = grid_id._id

                elif type(field) == type(ImageGridFsProxy()):
                    print "imagen"
                    if field.grid_id:
                        field.delete()

                    filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
                    field.put(upload, content_type=upload.content_type, filename=filename)
                
                setattr(instance, f.name, field)
            except AttributeError:
                # file was already uploaded and not changed during edit.
                # upload is already the gridfsproxy object we need.
                upload.get()
                setattr(instance, f.name, upload)
Example #8
0
    def post(self):

        form = AddItemForm()
        item = Item()

        categories = Category.objects.all()
        licenses = License.objects.all()
        form.set_categories(categories, g.lang)
        form.set_licenses(licenses)

        if form.validate_on_submit():
            # first, the user has to share something !
            if not form.github.data and not form.files.data:
                flash('Neither a repo URL nor files has been shared, come on!',
                      category='alert')
                return render_template('items/add_item.html', form=form)

            # give that something at least one title
            if not form.ar_title.data and not form.fr_title.data and \
               not form.en_title.data:

                flash('You need to give this item at least one title, \
                       just pick one language and name it!',
                      category='alert')
                return render_template('items/add_item.html', form=form)

            # now we can proceed
            ar_title = Title()
            fr_title = Title()
            en_title = Title()

            ar_title.title = form.ar_title.data.strip()
            ar_title.lang = 'ar'

            fr_title.title = form.fr_title.data.strip()
            fr_title.lang = 'fr'

            en_title.title = form.en_title.data.strip()
            en_title.lang = 'en'

            item.titles.append(ar_title)
            item.titles.append(fr_title)
            item.titles.append(en_title)

            item.description = form.description.data
            item.tags = form.tags.data.strip().split(',')
            item.category = Category.objects.get(
                category_id=int(form.category.data))

            item.submitter = User.objects.get(id=current_user.id)

            thumbnail = request.files['thumbnail']
            thumbnail_name = secure_filename(thumbnail.filename)

            if thumbnail and allowed_thumbnails(thumbnail_name):
                ext = thumbnail.mimetype.split('/')[-1]
                # use the 'thumbnail' name for all thumbnails
                filename = '.'.join(["thumbnail", ext])
                item.thumbnail.put(thumbnail.stream,
                                   content_type=thumbnail.mimetype,
                                   filename=filename)

            if form.github.data:
                item.github = form.github.data
                item.save()
                # no need to process any uploaded files
                flash('Item submitted successfully', category='success')
                return redirect(url_for('items.detail', item_id=item.item_id))
            else:
                item.license = License.objects.get(
                    license_id=int(form.license.data))

        else:
            flash('upload unsuccessful', category='error')
            return render_template('items/add_item.html', form=form)

        uploaded_files = request.files.getlist("files")
        for file in uploaded_files:
            # Make the filename safe, remove unsupported chars
            filename = secure_filename(file.filename)
            # Check if the file is one of the allowed types/extensions
            if file and allowed_file(filename):
                # put the file in the ListField.
                # see https://gist.github.com/tfausak/1299339
                file_ = GridFSProxy()
                file_.put(file.stream,
                          content_type=file.mimetype,
                          filename=filename)
                item.files.append(file_)
        # Save the thing
        item.save()
        flash('Item uploaded successfully', category='success')
        return redirect(url_for('items.detail', item_id=item.item_id))
Example #9
0
    def post(self, item_id):
        item = Item.objects.get_or_404(item_id=item_id)

        # only admins or the item submitter can edit the item
        if item.submitter.id != current_user.id:
            if not current_user.is_admin:
                abort(403)

        form = None
        if item.github:
            form = EditGithubItemForm()
        else:
            form = EditItemForm()
            licenses = License.objects.all()
            form.set_licenses(licenses)

        categories = Category.objects.all()
        form.set_categories(categories, g.lang)

        if form.validate_on_submit():
            for title in item.titles:  # ugly, I'll make it shorter, later...
                if title.lang == 'ar':
                    title.title = form.ar_title.data.strip()
                elif title.lang == 'en':
                    title.title = form.en_title.data.strip()
                else:
                    title.title = form.fr_title.data.strip()

            item.tags = form.tags.data.strip().split(',')
            item.category = Category.objects.get(
                category_id=int(form.category.data))

            if form.thumbnail.data:  # if the user has uploaded new thumbnail
                # remove the old one
                item.thumbnail.delete()
                # replace it with the new one
                thumbnail = request.files['thumbnail']
                thumbnail_name = secure_filename(thumbnail.filename)

                if thumbnail and allowed_thumbnails(thumbnail_name):
                    ext = thumbnail.mimetype.split('/')[-1]
                    # use the 'thumbnail' name for all thumbnails
                    filename = '.'.join(["thumbnail", ext])
                    item.thumbnail.put(thumbnail.stream,
                                       content_type=thumbnail.mimetype,
                                       filename=filename)

            if form.blog_post.data.strip():
                item.blog_post = form.blog_post.data
            if not item.github:
                item.description = form.description.data
                item.license = License.objects.get(
                    license_id=int(form.license.data))
            else:
                item.github = form.github.data
                item.save()
                # no need to process any uploaded files
                flash('Item updated successfully', category='success')
                return render_template('items/edit_item.html',
                                       form=form,
                                       item=item)

        else:
            flash("Couldn't update item", category='error')
            return render_template('items/edit_item.html',
                                   form=form,
                                   item=item)

        # now, replace them with the new ones
        uploaded_files = request.files.getlist("files")
        new_files = []
        for file in uploaded_files:
            # Make the filename safe, remove unsupported chars
            filename = secure_filename(file.filename)
            # Check if the file is one of the allowed types/extensions
            if file and allowed_file(filename):
                # put the file in the ListField.
                # see https://gist.github.com/tfausak/1299339
                file_ = GridFSProxy()
                file_.put(file.stream,
                          content_type=file.mimetype,
                          filename=filename)
                new_files.append(file_)
        if len(new_files) > 0:
            # delete old files first
            for file in item.files:
                file.delete()
            # push the new one
            item.files = new_files

        # Save the thing
        item.save()
        flash('Item updated successfully', category='success')
        return render_template('items/edit_item.html', form=form, item=item)
Example #10
0
 def get_fileslot(self, field):
     """Get an existing fileslot in a mapfield, or create it."""
     fileslot = field.get(self.version, GridFSProxy())
     field[self.version] = fileslot
     return fileslot
Example #11
0
    def post(self):

        form = AddItemForm()
        item = Item()

        categories = Category.objects.all()
        licenses = License.objects.all()
        form.set_categories(categories, g.lang)
        form.set_licenses(licenses)

        if form.validate_on_submit():
            # first, the user has to share something !
            if not form.github.data and not form.files.data:
                flash('Neither a repo URL nor files has been shared, come on!',
                      category='alert')
                return render_template('items/add_item.html', form=form)

            # give that something at least one title
            if not form.ar_title.data and not form.fr_title.data and \
               not form.en_title.data:

                flash('You need to give this item at least one title, \
                       just pick one language and name it!',
                      category='alert')
                return render_template('items/add_item.html', form=form)

            # now we can proceed
            ar_title = Title()
            fr_title = Title()
            en_title = Title()

            ar_title.title = form.ar_title.data.strip()
            ar_title.lang = 'ar'

            fr_title.title = form.fr_title.data.strip()
            fr_title.lang = 'fr'

            en_title.title = form.en_title.data.strip()
            en_title.lang = 'en'

            item.titles.append(ar_title)
            item.titles.append(fr_title)
            item.titles.append(en_title)

            item.description = form.description.data
            item.tags = form.tags.data.strip().split(',')
            item.category = Category.objects.get(category_id=
                                                 int(form.category.data))

            item.submitter = User.objects.get(id=current_user.id)

            thumbnail = request.files['thumbnail']
            thumbnail_name = secure_filename(thumbnail.filename)

            if thumbnail and allowed_thumbnails(thumbnail_name):
                ext = thumbnail.mimetype.split('/')[-1]
                # use the 'thumbnail' name for all thumbnails
                filename = '.'.join(["thumbnail", ext])
                item.thumbnail.put(thumbnail.stream,
                                   content_type=thumbnail.mimetype,
                                   filename=filename)

            if form.github.data:
                item.github = form.github.data
                item.save()
                # no need to process any uploaded files
                flash('Item submitted successfully', category='success')
                return redirect(url_for('items.detail', item_id=item.item_id))
            else:
                item.license = License.objects.get(license_id=
                                                   int(form.license.data))

        else:
            flash('upload unsuccessful', category='error')
            return render_template('items/add_item.html', form=form)

        uploaded_files = request.files.getlist("files")
        for file in uploaded_files:
            # Make the filename safe, remove unsupported chars
            filename = secure_filename(file.filename)
            # Check if the file is one of the allowed types/extensions
            if file and allowed_file(filename):
                # put the file in the ListField.
                # see https://gist.github.com/tfausak/1299339
                file_ = GridFSProxy()
                file_.put(file.stream,
                          content_type=file.mimetype,
                          filename=filename)
                item.files.append(file_)
        # Save the thing
        item.save()
        flash('Item uploaded successfully', category='success')
        return redirect(url_for('items.detail', item_id=item.item_id))
Example #12
0
    def post(self, item_id):
        item = Item.objects.get_or_404(item_id=item_id)

        # only admins or the item submitter can edit the item
        if item.submitter.id != current_user.id:
            if not current_user.is_admin:
                abort(403)

        form = None
        if item.github:
            form = EditGithubItemForm()
        else:
            form = EditItemForm()
            licenses = License.objects.all()
            form.set_licenses(licenses)

        categories = Category.objects.all()
        form.set_categories(categories, g.lang)

        if form.validate_on_submit():
            for title in item.titles:  # ugly, I'll make it shorter, later...
                if title.lang == 'ar':
                    title.title = form.ar_title.data.strip()
                elif title.lang == 'en':
                    title.title = form.en_title.data.strip()
                else:
                    title.title = form.fr_title.data.strip()

            item.tags = form.tags.data.strip().split(',')
            item.category = Category.objects.get(category_id=
                                                 int(form.category.data))

            if form.thumbnail.data:  # if the user has uploaded new thumbnail
                # remove the old one
                item.thumbnail.delete()
                # replace it with the new one
                thumbnail = request.files['thumbnail']
                thumbnail_name = secure_filename(thumbnail.filename)

                if thumbnail and allowed_thumbnails(thumbnail_name):
                    ext = thumbnail.mimetype.split('/')[-1]
                    # use the 'thumbnail' name for all thumbnails
                    filename = '.'.join(["thumbnail", ext])
                    item.thumbnail.put(thumbnail.stream,
                                       content_type=thumbnail.mimetype,
                                       filename=filename)

            if form.blog_post.data.strip():
                item.blog_post = form.blog_post.data
            if not item.github:
                item.description = form.description.data
                item.license = License.objects.get(license_id=
                                                   int(form.license.data))
            else:
                item.github = form.github.data
                item.save()
                # no need to process any uploaded files
                flash('Item updated successfully', category='success')
                return render_template('items/edit_item.html', form=form,
                                       item=item)

        else:
            flash("Couldn't update item", category='error')
            return render_template('items/edit_item.html', form=form,
                                   item=item)

        # now, replace them with the new ones
        uploaded_files = request.files.getlist("files")
        new_files = []
        for file in uploaded_files:
            # Make the filename safe, remove unsupported chars
            filename = secure_filename(file.filename)
            # Check if the file is one of the allowed types/extensions
            if file and allowed_file(filename):
                # put the file in the ListField.
                # see https://gist.github.com/tfausak/1299339
                file_ = GridFSProxy()
                file_.put(file.stream,
                          content_type=file.mimetype,
                          filename=filename)
                new_files.append(file_)
        if len(new_files) > 0:
            # delete old files first
            for file in item.files:
                file.delete()
            # push the new one
            item.files = new_files

        # Save the thing
        item.save()
        flash('Item updated successfully', category='success')
        return render_template('items/edit_item.html', form=form,
                               item=item)
Example #13
0
def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
    """
    Constructs and returns a document instance from the bound ``form``'s
    ``cleaned_data``, but does not save the returned instance to the
    database.
    """
    cleaned_data = form.cleaned_data
    file_field_list = []

    # check wether object is instantiated
    if isinstance(instance, type):
        instance = instance()

    for f in instance._fields.values():
        if isinstance(f, ObjectIdField):
            continue
        if not f.name in cleaned_data:
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        # Defer saving file-type fields until after the other fields, so a
        # callable upload_to can use the values from other fields.
        if isinstance(f, FileField) or (isinstance(f, (MapField, ListField))
                                        and isinstance(f.field, FileField)):
            file_field_list.append(f)
        else:
            setattr(instance, f.name, cleaned_data.get(f.name))

    for f in file_field_list:
        if isinstance(f, MapField):
            map_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for key, uploaded_file in uploads.items():
                if uploaded_file is None:
                    continue
                file_data = map_field.get(key, None)
                map_field[key] = _save_iterator_file(f, uploaded_file,
                                                     file_data)
            setattr(instance, f.name, map_field)
        elif isinstance(f, ListField):
            list_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for i, uploaded_file in enumerate(uploads):
                if uploaded_file is None:
                    continue
                try:
                    file_data = list_field[i]
                except IndexError:
                    file_data = None
                file_obj = _save_iterator_file(f, uploaded_file, file_data)
                try:
                    list_field[i] = file_obj
                except IndexError:
                    list_field.append(file_obj)
            setattr(instance, f.name, list_field)
        else:
            field = getattr(instance, f.name)
            upload = cleaned_data[f.name]
            if upload is None:
                continue

            try:
                print "el tipo en field es %s" % type(field)

                upload.file.seek(0)
                # delete first to get the names right
                #if field.grid_id:
                #    field.delete()
                from mongoengine.fields import GridFSProxy, ImageGridFsProxy
                if type(field) == type(GridFSProxy()):
                    print "fichero"
                    if field.grid_id:
                        field = GridFSProxy()

                    import hashlib
                    try:
                        grid_id = _get_grid_id(
                            hashlib.md5(upload.file.read()).hexdigest(),
                            f.db_alias, f.collection_name)
                    except Exception, e:
                        print "Error: %s" % e

                    if grid_id == None:
                        upload.file.seek(0)
                        filename = _get_unique_filename(
                            upload.name, f.db_alias, f.collection_name)
                        field.put(upload,
                                  content_type=upload.content_type,
                                  filename=filename)
                    else:
                        field.grid_id = grid_id._id

                elif type(field) == type(ImageGridFsProxy()):
                    print "imagen"
                    if field.grid_id:
                        field.delete()

                    filename = _get_unique_filename(upload.name, f.db_alias,
                                                    f.collection_name)
                    field.put(upload,
                              content_type=upload.content_type,
                              filename=filename)

                setattr(instance, f.name, field)
            except AttributeError:
                # file was already uploaded and not changed during edit.
                # upload is already the gridfsproxy object we need.
                upload.get()
                setattr(instance, f.name, upload)
Example #14
0
 def grid_proxy(self):
     if not self._grid_proxy:
         self._grid_proxy = GridFSProxy(db_alias=self.db_alias,
                                        collection_name=self.collection)
     return self._grid_proxy