def upload():
logger = current_app.logger
"""
The upload method takes a uploaded file and puts it into
the celery processing queue using the Redis backend. Filename
integrity is enforced by renaming the uploaded file to a unique
name and deleting it after successfull processing.
"""
if request.method == 'POST':
# FIXME: convert this to WTForms
meshlab = request.form.getlist('meshlab')
template = request.form['template']
if 'email_to' in request.form:
email_to = request.form['email_to']
else:
email_to=None
file = request.files['file']
metadata = request.files['metadata']
url = request.form['url']
# options to pass to convertion task
options = dict()
# create a UUID like hash for temporary file storage
hash = uuid.uuid4().hex
options.update(hash=hash)
# This whole section is kind of flimsy.
# - code more idiomatic
# - download should be performed asynchrounously, though some checks
# should be performed here (allowed hosts, filenames, etc.)
# - downloading a url triggered via public web form is a HUGE security
# risk. Basically anyone can kill our sever by pasting a url to be
# downloaded. Therefore, at the moment, the ULRs are restricted to
# the ALLOWED_DOWNLOAD_HOSTS settings.
# error handling can be done smarter
# URL instead of file
if url:
# basic security
if not security.is_allowed_host(url):
flash("Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])), 'error')
return render_template('frontend/index.html')
# download file to disk
r = requests.get(url, stream=True, verify=False)
filename = secure_filename(os.path.split(url)[-1].split("?")[0])
# FIXME: this should check the mimetype in the http response header
# as well
if not security.is_allowed_file(filename):
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
flash("File too big. Please don't upload files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])), 'error')
return render_template('frontend/index.html')
else:
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, filename)
with open(filename, "wb") as data:
data.write(r.content)
else:
flash("Could not download file {0} Status code: {1}".format(url, r.status_code), 'error')
return render_template('frontend/index.html')
else:
# in case of file upload place the uploaded file in a folder
# named <uuid>
if file and security.is_allowed_file(file.filename):
filename = secure_filename(file.filename)
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, file.filename)
file.save(filename)
# in case the user uploaded a meta file, store this as well
# FIXME make sure only processed when valid template selection
if metadata and not compression.is_archive(filename):
meta_filename = os.path.join(upload_directory, 'metadata' + os.path.splitext(metadata.filename)[1])
metadata.save(meta_filename)
# options for task
options.update(meta_filename=meta_filename)
else:
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if email_to:
# we need to add at least captcha system to protect from
# spammers, for now setting the sender env var enables the
# email system, use with care behind pw protected
if current_app.config['DEFAULT_MAIL_SENDER'] == 'noreply@localhost':
options.update(email_to=None)
else:
options.update(email_to=email_to)
if meshlab:
options.update(meshlab=meshlab)
options.update(
template=template
)
retval = tasks.convert_model.apply_async((filename, options))
return redirect(url_for('frontend.status', task_id=retval.task_id))
return render_template('frontend/index.html')
def add_bucket():
"""
This methods allows for uploading arbitrary files to the server
which later are used for processing::
POST /buckets
The HTTP request requires a filename header to identify the file::
X-Filename: herkules.ply
Additionally the Content-Type header needs to be set to::
Content-Type: application/octet-stream
You can then stream binary content in the body.
Note that you have to specify a filename even if you stream the
file directly from an application. This is required in case you
want to upload other assets to the bucket which are name dependent
(for exmaple when using the metadata template).
After successful upload, a bucket ID among with a status message and
the filename under which the data is stored within he bucket.
"""
# create a UUID like hash for temporary file storage (bucket id)
hash = uuid.uuid4().hex
if not request.headers['Content-Type'] == 'application/octet-stream':
resp = jsonify(message="Unsupported media type. Maybe you forgot to set X-Filename header?")
resp.status_code = 415 # bad resquest
return resp
if not 'X-Filename' in request.headers:
resp = jsonify(message="You probalby forgot to set X-Filename header?")
resp.status_code = 415 # bad resquest
return resp
if not request.data:
resp = jsonify(message="Please support data with your request?")
resp.status_code = 415 # bad resquest
return resp
filename = request.headers['X-Filename']
if security.is_allowed_file(filename):
filename = werkzeug.secure_filename(filename)
ofilename = filename
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, filename)
try:
with open(filename, "wb") as data:
data.write(request.data)
resp = jsonify(
message="Payload dropped sucessfully",
bucket_id=hash,
filename=ofilename,
)
resp.status_code = 201 # created
return resp
except IOError:
current_app.logger.error("Can not save {0}. Abort.".format(filename))
resp = jsonify(message="Problem saving uploaded file")
resp.status_code = 500 # server error
return resp
else:
resp = jsonify(message="Filename unsecure, upload rejected.")
resp.status_code = 420 # policy not fullfilled
return resp
def add_job():
"""
Adding a job to the processing queue either by fetching data from
a known source or by using a previously created bucket full off
resources (see ..)::
{
"payload": "bucket://1234afdsafdsfdsa232",
// for the moment you also need to specify the filename you want
"payload_filename: "herkules.ply"
// alternatively, to load from a URI use this instead:
"payload": "http://someplace.zip",
// all the following are optional:
"email_to": "*****@*****.**",
// array of strings representing meshlab filters. leave out
// "meshlab" entry at all if no meshalb pre-processing is desired
"meshlab": [
"Remove Duplicate Faces",
"Remove Duplicated Vertex",
"Remove Zero Area Faces",
"Remove Isolated pieces (wrt Face Num.)",
"Remove Unreferenced Vertex",
"Extract Information"
]
// one out of the list of names you get with /bundles
// always use the "name" field you get from GET /bundles as the name
// might change. You can cache the names locally but be sure to expire
// once in a while.
"template": "basic",
// the bundle name to be used for this job
// in the future its also possible to override templte specific settings and options (shown but noop)
//"bundle": {
// "name": "modelconvert.bundles.pop", // this can also contain a bundle spec and related data
// "settings": {
// "aopt.pop": true,
// "aopt.command": "{command} {input} {output} -what -ever={0} -is -required",
// "meshlab.enabled": false,
// }
// }
}
For exmaple a simple payload to convert a single model without meshalb
sourced from a URL could look like this::
{
"payload": "http://domain.tld/model.obj",
"template": "basic"
}
In return you will get a json response with various data about
your request::
{
// clear text informational message, HTTP status code serves as numeric indicator
"message": "Job accepted with ID 123",
// the task ID the job is running on
"task_id": "123",
// poll URI for checking less frequently for results
"job_url": "full.host/v1/jobs/123",
// URI for status updates through push protocl. This implements
// the W3C EventSource specification. So your client needs to
// support this in order to reciece push updates.
"progress_url": "full.host/v1/stream/123",
}
"""
options = dict() # options passted to task
data = request.json
if not data or not 'payload' in data:
resp = jsonify(message="No payload provided. You need to specify what to convert.")
resp.status_code = 400 # bad resquest
return resp
url = urlparse.urlparse(data['payload'])
current_app.logger.info("Found {0} payload".format(url.scheme))
if url.scheme == 'http':
#
# FIXME BIGTIME
# THE URL DOWNLOADING SHOULD OCCUR IN THE TASK AND NOT BLOCK
# THE INTERFACE
#
if not security.is_allowed_host(data['payload']):
resp = jsonify(message="Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url.netloc, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])))
resp.status_code = 403 #forbidden
return resp
# download file to disk
r = requests.get(data['payload'], stream=True, verify=False)
filename = werkzeug.secure_filename(os.path.split(data['payload'])[-1].split("?")[0])
# FIXME: this should check the mimetype in the http response header
# as well
if not security.is_allowed_file(filename):
resp = jsonify(message="Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']))
resp.status_code = 403 #forbidden
return resp
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
resp = jsonify(message="File too big. Please don't try to use files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])))
resp.status_code = 416 # request range unsatifieable
return resp
else:
# create a UUID like hash for temporary file storage
hash = uuid.uuid4().hex
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, filename)
with open(filename, "wb") as fdata:
fdata.write(r.content)
options.update(hash=hash)
else:
resp = jsonify(message="Could not download file {0} Status code: {1}".format(data['payload'], r.status_code))
resp.status_code = 416 # request range unsatifieable
return resp
elif url.scheme == 'bucket':
if not 'payload_filename' in data:
resp = jsonify(message="Please specify the payload_filename attribute in your request.")
resp.status_code = 400 # bad
return resp
current_app.logger.debug("Using Bucket ID: {0} with entry point filename".format(url.netloc, filename))
options.update(hash=url.netloc)
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], url.netloc)
filename = os.path.join(upload_directory, data['payload_filename'])
def upload():
"""
The upload method takes a uploaded file and puts it into
the celery processing queue using the Redis backend. Filename
integrity is enforced by renaming the uploaded file to a unique
name and deleting it after successfull processing.
"""
if request.method == 'POST':
# FIXME: convert this to WTForms
meshlab = request.form.getlist('meshlab')
aopt = request.form['aopt']
template = request.form['template']
file = request.files['file']
url = request.form['url']
# options to pass to convertion task
options = dict()
# create a UUID like hash for temporary file storage
#hash = "%032x" % random.getrandbits(128)
hash = uuid.uuid4().hex
options.update(hash=hash)
# This whole section is kind of flimsy.
# - download should be performed asynchrounously, though some checks
# should be performed here (allowed hosts, filenames, etc.)
# - downloading a url triggered via public web form is a HUGE security
# risk. Basically anyone can kill our sever by pasting a url to be
# downloaded. Therefore, at the moment, the ULRs are restricted to
# the ALLOWED_DOWNLOAD_HOSTS settings.
# error handling can be done smarter
# URL instead of file
if url:
# basic security
if not security.is_allowed_host(url):
flash("Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])), 'error')
return render_template('frontend/index.html')
# download file to disk
r = requests.get(url, stream=True, verify=False)
filename = secure_filename(os.path.split(url)[-1].split("?")[0])
filename = os.path.join(current_app.config['UPLOAD_PATH'], filename)
if not security.is_allowed_file(filename):
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
flash("File too big. Please don't upload files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])), 'error')
return render_template('frontend/index.html')
else:
with open(filename, "wb") as data:
data.write(r.content)
else:
flash("Could not download file {0} Status code: {1}".format(url, r.status_code), 'error')
return render_template('frontend/index.html')
else:
# in case of file upload
if file and security.is_allowed_file(file.filename):
filename = secure_filename(file.filename)
# anonymize filename, keep extension and save
filename = os.path.join(current_app.config['UPLOAD_PATH'],
hash + os.path.splitext(file.filename)[1])
file.save(filename)
else:
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if meshlab:
options.update(meshlab=meshlab)
# in case the user uploaded a meta file, store this as well
# FIXME make sure only processed when valid template selection
# (DoS)
metadata = request.files['metadata']
if metadata:
meta_filename = os.path.join(current_app.config['UPLOAD_PATH'], hash, 'metadata' + os.path.splitext(metadata.filename)[1])
metadata.save(meta_filename)
options.update(meta_filename=meta_filename)
options.update(
aopt=aopt,
template=template
)
retval = tasks.convert_model.apply_async((filename, options))
return redirect(url_for('frontend.status', task_id=retval.task_id))
return render_template('frontend/index.html')
