def post(self):
form = AddItemForm()
item = Item()
if form.validate_on_submit():
ar_title = Titles()
fr_title = Titles()
en_title = Titles()
ar_title.title = form.ar_title.data.strip()
ar_title.lang = 'ar'
fr_title.title = form.fr_title.data.strip()
fr_title.lang = 'fr'
en_title.title = form.en_title.data.strip()
en_title.lang = 'en'
item.titles.append(ar_title)
item.titles.append(fr_title)
item.titles.append(en_title)
item.description = form.description.data
item.submitter = User.objects.get(id=current_user.id)
else:
flash('upload unsuccessful', 'error')
return render_template('items/add_item.html', form=form)
uploaded_files = request.files.getlist("files")
thumbnail = request.files['thumbnail']
thumbnail_name = secure_filename(thumbnail.filename)
if thumbnail and allowed_thumbnails(thumbnail_name):
ext = thumbnail.mimetype.split('/')[-1]
# use the 'thumbnail' name for all thumbnails
filename = '.'.join(["thumbnail", ext])
item.thumbnail.put(thumbnail.stream,
content_type=thumbnail.mimetype,
filename=filename)
for file in uploaded_files:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
# Check if the file is one of the allowed types/extensions
if file and allowed_file(filename):
# put the file in the ListField.
# see https://gist.github.com/tfausak/1299339
file_ = GridFSProxy()
file_.put(file.stream,
content_type=file.mimetype,
filename=filename)
item.files.append(file_)
# Save the thing
item.save()
flash('upload successful')
return render_template('items/add_item.html', form=form)
def put(self, obj, name, attributes=None):
"""
Store a NotebookNode
:param obj: the NotebookNode to store
:param name: the name of the notebook
"""
if not name.endswith('.ipynb'):
name += '.ipynb'
sbuf = StringIO()
bbuf = BytesIO()
# nbwrite expects string, fs.put expects bytes
nbwrite(obj, sbuf, version=4)
sbuf.seek(0)
bbuf.write(sbuf.getvalue().encode('utf8'))
bbuf.seek(0)
# see if we have a file already, if so replace the gridfile
meta = self.store.metadata(name)
if not meta:
filename = uuid4().hex
fileid = self._fs.put(bbuf, filename=filename)
meta = self.store._make_metadata(
name=name,
prefix=self.store.prefix,
bucket=self.store.bucket,
kind=self.kind,
attributes=attributes,
gridfile=GridFSProxy(grid_id=fileid))
else:
meta.gridfile.replace(bbuf)
return meta.save()
def put(self, obj, name, attributes=None):
"""
Store a NotebookNode
:param obj: the NotebookNode to store
:param name: the name of the notebook
"""
if not name.endswith('.ipynb'):
name += '.ipynb'
sbuf = StringIO()
bbuf = BytesIO()
# nbwrite expects string, fs.put expects bytes
nbwrite(obj, sbuf, version=4)
sbuf.seek(0)
bbuf.write(sbuf.getvalue().encode('utf8'))
bbuf.seek(0)
# see if we have a file already, if so replace the gridfile
meta = self.store.metadata(name)
if not meta:
filename = uuid4().hex
fileid = self._fs.put(bbuf, filename=filename)
meta = self.store._make_metadata(
name=name,
prefix=self.store.prefix,
bucket=self.store.bucket,
kind=self.kind,
attributes=attributes,
gridfile=GridFSProxy(grid_id=fileid))
meta = meta.save()
else:
filename = uuid4().hex
meta.gridfile.delete()
fileid = self._fs.put(bbuf, filename=filename)
meta.gridfile = GridFSProxy(grid_id=fileid)
meta = meta.save()
# set config
nb_config = self.get_notebook_config(name)
meta_config = meta.attributes.get('config', {})
if nb_config:
meta_config.update(dict(**nb_config))
meta.attributes['config'] = meta_config
meta = meta.save()
if not name.startswith('results') and ('run-at' in meta_config):
meta = self.schedule(name)
return meta
def upload(space='main', doc_id=None):
print 'enter request'
if doc_id is None:
redirect(url_for('.dashboard'))
else:
doc = BaseDocument.objects(id=doc_id).first()
# check acl for document
acl.apply('write', doc.acl, _('document'))
print request.files
file = request.files['upload']
file_ = GridFSProxy()
file_.put(file.read(), content_type=file.content_type, filename=file.filename)
doc.attachments.append(file_)
doc.save()
flash(_('File uploaded successfully.'), 'info')
return redirect(url_for('.view', space=space, doc_id=doc.id))
def put_dataframe_as_hdf(self, obj, name, attributes=None):
filename = self._get_obj_store_key(name, '.hdf')
hdffname = self._package_dataframe2hdf(obj, filename)
with open(hdffname, 'rb') as fhdf:
fileid = self.fs.put(fhdf, filename=filename)
return self._make_metadata(name=name,
prefix=self.prefix,
bucket=self.bucket,
kind=MDREGISTRY.PANDAS_HDF,
attributes=attributes,
gridfile=GridFSProxy(grid_id=fileid)).save()
def put_model(self, obj, name, attributes=None):
"""
Packages a model using joblib and stores in GridFS
"""
zipfname = self._package_model(obj, name)
with open(zipfname, 'rb') as fzip:
fileid = self.model_store.fs.put(
fzip, filename=self.model_store._get_obj_store_key(name, 'omm'))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
return self.model_store._make_metadata(
name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=MDREGISTRY.SKLEARN_JOBLIB,
attributes=attributes,
gridfile=gridfile).save()
def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
"""
Constructs and returns a document instance from the bound ``form``'s
``cleaned_data``, but does not save the returned instance to the
database.
"""
cleaned_data = form.cleaned_data
file_field_list = []
# check wether object is instantiated
if isinstance(instance, type):
instance = instance()
for f in instance._fields.values():
if isinstance(f, ObjectIdField):
continue
if not f.name in cleaned_data:
continue
if fields is not None and f.name not in fields:
continue
if exclude and f.name in exclude:
continue
# Defer saving file-type fields until after the other fields, so a
# callable upload_to can use the values from other fields.
if isinstance(f, FileField) or (isinstance(f, (MapField, ListField)) and isinstance(f.field, FileField)):
file_field_list.append(f)
else:
setattr(instance, f.name, cleaned_data.get(f.name))
for f in file_field_list:
if isinstance(f, MapField):
map_field = getattr(instance, f.name)
uploads = cleaned_data[f.name]
for key, uploaded_file in uploads.items():
if uploaded_file is None:
continue
file_data = map_field.get(key, None)
map_field[key] = _save_iterator_file(f, uploaded_file, file_data)
setattr(instance, f.name, map_field)
elif isinstance(f, ListField):
list_field = getattr(instance, f.name)
uploads = cleaned_data[f.name]
for i, uploaded_file in enumerate(uploads):
if uploaded_file is None:
continue
try:
file_data = list_field[i]
except IndexError:
file_data = None
file_obj = _save_iterator_file(f, uploaded_file, file_data)
try:
list_field[i] = file_obj
except IndexError:
list_field.append(file_obj)
setattr(instance, f.name, list_field)
else:
field = getattr(instance, f.name)
upload = cleaned_data[f.name]
if upload is None:
continue
try:
print "el tipo en field es %s" % type(field)
upload.file.seek(0)
# delete first to get the names right
#if field.grid_id:
# field.delete()
from mongoengine.fields import GridFSProxy, ImageGridFsProxy
if type(field) == type(GridFSProxy()):
print "fichero"
if field.grid_id:
field = GridFSProxy()
import hashlib
try:
grid_id = _get_grid_id(hashlib.md5(upload.file.read()).hexdigest(), f.db_alias, f.collection_name)
except Exception, e:
print "Error: %s" % e
if grid_id == None:
upload.file.seek(0)
filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
field.put(upload, content_type=upload.content_type, filename=filename)
else:
field.grid_id = grid_id._id
elif type(field) == type(ImageGridFsProxy()):
print "imagen"
if field.grid_id:
field.delete()
filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
field.put(upload, content_type=upload.content_type, filename=filename)
setattr(instance, f.name, field)
except AttributeError:
# file was already uploaded and not changed during edit.
# upload is already the gridfsproxy object we need.
upload.get()
setattr(instance, f.name, upload)
def post(self):
form = AddItemForm()
item = Item()
categories = Category.objects.all()
licenses = License.objects.all()
form.set_categories(categories, g.lang)
form.set_licenses(licenses)
if form.validate_on_submit():
# first, the user has to share something !
if not form.github.data and not form.files.data:
flash('Neither a repo URL nor files has been shared, come on!',
category='alert')
return render_template('items/add_item.html', form=form)
# give that something at least one title
if not form.ar_title.data and not form.fr_title.data and \
not form.en_title.data:
flash('You need to give this item at least one title, \
just pick one language and name it!',
category='alert')
return render_template('items/add_item.html', form=form)
# now we can proceed
ar_title = Title()
fr_title = Title()
en_title = Title()
ar_title.title = form.ar_title.data.strip()
ar_title.lang = 'ar'
fr_title.title = form.fr_title.data.strip()
fr_title.lang = 'fr'
en_title.title = form.en_title.data.strip()
en_title.lang = 'en'
item.titles.append(ar_title)
item.titles.append(fr_title)
item.titles.append(en_title)
item.description = form.description.data
item.tags = form.tags.data.strip().split(',')
item.category = Category.objects.get(
category_id=int(form.category.data))
item.submitter = User.objects.get(id=current_user.id)
thumbnail = request.files['thumbnail']
thumbnail_name = secure_filename(thumbnail.filename)
if thumbnail and allowed_thumbnails(thumbnail_name):
ext = thumbnail.mimetype.split('/')[-1]
# use the 'thumbnail' name for all thumbnails
filename = '.'.join(["thumbnail", ext])
item.thumbnail.put(thumbnail.stream,
content_type=thumbnail.mimetype,
filename=filename)
if form.github.data:
item.github = form.github.data
item.save()
# no need to process any uploaded files
flash('Item submitted successfully', category='success')
return redirect(url_for('items.detail', item_id=item.item_id))
else:
item.license = License.objects.get(
license_id=int(form.license.data))
else:
flash('upload unsuccessful', category='error')
return render_template('items/add_item.html', form=form)
uploaded_files = request.files.getlist("files")
for file in uploaded_files:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
# Check if the file is one of the allowed types/extensions
if file and allowed_file(filename):
# put the file in the ListField.
# see https://gist.github.com/tfausak/1299339
file_ = GridFSProxy()
file_.put(file.stream,
content_type=file.mimetype,
filename=filename)
item.files.append(file_)
# Save the thing
item.save()
flash('Item uploaded successfully', category='success')
return redirect(url_for('items.detail', item_id=item.item_id))
def post(self, item_id):
item = Item.objects.get_or_404(item_id=item_id)
# only admins or the item submitter can edit the item
if item.submitter.id != current_user.id:
if not current_user.is_admin:
abort(403)
form = None
if item.github:
form = EditGithubItemForm()
else:
form = EditItemForm()
licenses = License.objects.all()
form.set_licenses(licenses)
categories = Category.objects.all()
form.set_categories(categories, g.lang)
if form.validate_on_submit():
for title in item.titles: # ugly, I'll make it shorter, later...
if title.lang == 'ar':
title.title = form.ar_title.data.strip()
elif title.lang == 'en':
title.title = form.en_title.data.strip()
else:
title.title = form.fr_title.data.strip()
item.tags = form.tags.data.strip().split(',')
item.category = Category.objects.get(
category_id=int(form.category.data))
if form.thumbnail.data: # if the user has uploaded new thumbnail
# remove the old one
item.thumbnail.delete()
# replace it with the new one
thumbnail = request.files['thumbnail']
thumbnail_name = secure_filename(thumbnail.filename)
if thumbnail and allowed_thumbnails(thumbnail_name):
ext = thumbnail.mimetype.split('/')[-1]
# use the 'thumbnail' name for all thumbnails
filename = '.'.join(["thumbnail", ext])
item.thumbnail.put(thumbnail.stream,
content_type=thumbnail.mimetype,
filename=filename)
if form.blog_post.data.strip():
item.blog_post = form.blog_post.data
if not item.github:
item.description = form.description.data
item.license = License.objects.get(
license_id=int(form.license.data))
else:
item.github = form.github.data
item.save()
# no need to process any uploaded files
flash('Item updated successfully', category='success')
return render_template('items/edit_item.html',
form=form,
item=item)
else:
flash("Couldn't update item", category='error')
return render_template('items/edit_item.html',
form=form,
item=item)
# now, replace them with the new ones
uploaded_files = request.files.getlist("files")
new_files = []
for file in uploaded_files:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
# Check if the file is one of the allowed types/extensions
if file and allowed_file(filename):
# put the file in the ListField.
# see https://gist.github.com/tfausak/1299339
file_ = GridFSProxy()
file_.put(file.stream,
content_type=file.mimetype,
filename=filename)
new_files.append(file_)
if len(new_files) > 0:
# delete old files first
for file in item.files:
file.delete()
# push the new one
item.files = new_files
# Save the thing
item.save()
flash('Item updated successfully', category='success')
return render_template('items/edit_item.html', form=form, item=item)
def get_fileslot(self, field):
"""Get an existing fileslot in a mapfield, or create it."""
fileslot = field.get(self.version, GridFSProxy())
field[self.version] = fileslot
return fileslot
def post(self):
form = AddItemForm()
item = Item()
categories = Category.objects.all()
licenses = License.objects.all()
form.set_categories(categories, g.lang)
form.set_licenses(licenses)
if form.validate_on_submit():
# first, the user has to share something !
if not form.github.data and not form.files.data:
flash('Neither a repo URL nor files has been shared, come on!',
category='alert')
return render_template('items/add_item.html', form=form)
# give that something at least one title
if not form.ar_title.data and not form.fr_title.data and \
not form.en_title.data:
flash('You need to give this item at least one title, \
just pick one language and name it!',
category='alert')
return render_template('items/add_item.html', form=form)
# now we can proceed
ar_title = Title()
fr_title = Title()
en_title = Title()
ar_title.title = form.ar_title.data.strip()
ar_title.lang = 'ar'
fr_title.title = form.fr_title.data.strip()
fr_title.lang = 'fr'
en_title.title = form.en_title.data.strip()
en_title.lang = 'en'
item.titles.append(ar_title)
item.titles.append(fr_title)
item.titles.append(en_title)
item.description = form.description.data
item.tags = form.tags.data.strip().split(',')
item.category = Category.objects.get(category_id=
int(form.category.data))
item.submitter = User.objects.get(id=current_user.id)
thumbnail = request.files['thumbnail']
thumbnail_name = secure_filename(thumbnail.filename)
if thumbnail and allowed_thumbnails(thumbnail_name):
ext = thumbnail.mimetype.split('/')[-1]
# use the 'thumbnail' name for all thumbnails
filename = '.'.join(["thumbnail", ext])
item.thumbnail.put(thumbnail.stream,
content_type=thumbnail.mimetype,
filename=filename)
if form.github.data:
item.github = form.github.data
item.save()
# no need to process any uploaded files
flash('Item submitted successfully', category='success')
return redirect(url_for('items.detail', item_id=item.item_id))
else:
item.license = License.objects.get(license_id=
int(form.license.data))
else:
flash('upload unsuccessful', category='error')
return render_template('items/add_item.html', form=form)
uploaded_files = request.files.getlist("files")
for file in uploaded_files:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
# Check if the file is one of the allowed types/extensions
if file and allowed_file(filename):
# put the file in the ListField.
# see https://gist.github.com/tfausak/1299339
file_ = GridFSProxy()
file_.put(file.stream,
content_type=file.mimetype,
filename=filename)
item.files.append(file_)
# Save the thing
item.save()
flash('Item uploaded successfully', category='success')
return redirect(url_for('items.detail', item_id=item.item_id))
def post(self, item_id):
item = Item.objects.get_or_404(item_id=item_id)
# only admins or the item submitter can edit the item
if item.submitter.id != current_user.id:
if not current_user.is_admin:
abort(403)
form = None
if item.github:
form = EditGithubItemForm()
else:
form = EditItemForm()
licenses = License.objects.all()
form.set_licenses(licenses)
categories = Category.objects.all()
form.set_categories(categories, g.lang)
if form.validate_on_submit():
for title in item.titles: # ugly, I'll make it shorter, later...
if title.lang == 'ar':
title.title = form.ar_title.data.strip()
elif title.lang == 'en':
title.title = form.en_title.data.strip()
else:
title.title = form.fr_title.data.strip()
item.tags = form.tags.data.strip().split(',')
item.category = Category.objects.get(category_id=
int(form.category.data))
if form.thumbnail.data: # if the user has uploaded new thumbnail
# remove the old one
item.thumbnail.delete()
# replace it with the new one
thumbnail = request.files['thumbnail']
thumbnail_name = secure_filename(thumbnail.filename)
if thumbnail and allowed_thumbnails(thumbnail_name):
ext = thumbnail.mimetype.split('/')[-1]
# use the 'thumbnail' name for all thumbnails
filename = '.'.join(["thumbnail", ext])
item.thumbnail.put(thumbnail.stream,
content_type=thumbnail.mimetype,
filename=filename)
if form.blog_post.data.strip():
item.blog_post = form.blog_post.data
if not item.github:
item.description = form.description.data
item.license = License.objects.get(license_id=
int(form.license.data))
else:
item.github = form.github.data
item.save()
# no need to process any uploaded files
flash('Item updated successfully', category='success')
return render_template('items/edit_item.html', form=form,
item=item)
else:
flash("Couldn't update item", category='error')
return render_template('items/edit_item.html', form=form,
item=item)
# now, replace them with the new ones
uploaded_files = request.files.getlist("files")
new_files = []
for file in uploaded_files:
# Make the filename safe, remove unsupported chars
filename = secure_filename(file.filename)
# Check if the file is one of the allowed types/extensions
if file and allowed_file(filename):
# put the file in the ListField.
# see https://gist.github.com/tfausak/1299339
file_ = GridFSProxy()
file_.put(file.stream,
content_type=file.mimetype,
filename=filename)
new_files.append(file_)
if len(new_files) > 0:
# delete old files first
for file in item.files:
file.delete()
# push the new one
item.files = new_files
# Save the thing
item.save()
flash('Item updated successfully', category='success')
return render_template('items/edit_item.html', form=form,
item=item)
def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
"""
Constructs and returns a document instance from the bound ``form``'s
``cleaned_data``, but does not save the returned instance to the
database.
"""
cleaned_data = form.cleaned_data
file_field_list = []
# check wether object is instantiated
if isinstance(instance, type):
instance = instance()
for f in instance._fields.values():
if isinstance(f, ObjectIdField):
continue
if not f.name in cleaned_data:
continue
if fields is not None and f.name not in fields:
continue
if exclude and f.name in exclude:
continue
# Defer saving file-type fields until after the other fields, so a
# callable upload_to can use the values from other fields.
if isinstance(f, FileField) or (isinstance(f, (MapField, ListField))
and isinstance(f.field, FileField)):
file_field_list.append(f)
else:
setattr(instance, f.name, cleaned_data.get(f.name))
for f in file_field_list:
if isinstance(f, MapField):
map_field = getattr(instance, f.name)
uploads = cleaned_data[f.name]
for key, uploaded_file in uploads.items():
if uploaded_file is None:
continue
file_data = map_field.get(key, None)
map_field[key] = _save_iterator_file(f, uploaded_file,
file_data)
setattr(instance, f.name, map_field)
elif isinstance(f, ListField):
list_field = getattr(instance, f.name)
uploads = cleaned_data[f.name]
for i, uploaded_file in enumerate(uploads):
if uploaded_file is None:
continue
try:
file_data = list_field[i]
except IndexError:
file_data = None
file_obj = _save_iterator_file(f, uploaded_file, file_data)
try:
list_field[i] = file_obj
except IndexError:
list_field.append(file_obj)
setattr(instance, f.name, list_field)
else:
field = getattr(instance, f.name)
upload = cleaned_data[f.name]
if upload is None:
continue
try:
print "el tipo en field es %s" % type(field)
upload.file.seek(0)
# delete first to get the names right
#if field.grid_id:
# field.delete()
from mongoengine.fields import GridFSProxy, ImageGridFsProxy
if type(field) == type(GridFSProxy()):
print "fichero"
if field.grid_id:
field = GridFSProxy()
import hashlib
try:
grid_id = _get_grid_id(
hashlib.md5(upload.file.read()).hexdigest(),
f.db_alias, f.collection_name)
except Exception, e:
print "Error: %s" % e
if grid_id == None:
upload.file.seek(0)
filename = _get_unique_filename(
upload.name, f.db_alias, f.collection_name)
field.put(upload,
content_type=upload.content_type,
filename=filename)
else:
field.grid_id = grid_id._id
elif type(field) == type(ImageGridFsProxy()):
print "imagen"
if field.grid_id:
field.delete()
filename = _get_unique_filename(upload.name, f.db_alias,
f.collection_name)
field.put(upload,
content_type=upload.content_type,
filename=filename)
setattr(instance, f.name, field)
except AttributeError:
# file was already uploaded and not changed during edit.
# upload is already the gridfsproxy object we need.
upload.get()
setattr(instance, f.name, upload)
def grid_proxy(self):
if not self._grid_proxy:
self._grid_proxy = GridFSProxy(db_alias=self.db_alias,
collection_name=self.collection)
return self._grid_proxy