def add_job():
"""
Adding a job to the processing queue either by fetching data from
a known source or by using a previously created bucket full off
resources (see ..)::
{
"payload": "bucket://1234afdsafdsfdsa232",
// for the moment you also need to specify the filename you want
"payload_filename: "herkules.ply"
// alternatively, to load from a URI use this instead:
"payload": "http://someplace.zip",
// all the following are optional:
"email_to": "*****@*****.**",
// array of strings representing meshlab filters. leave out
// "meshlab" entry at all if no meshalb pre-processing is desired
"meshlab": [
"Remove Duplicate Faces",
"Remove Duplicated Vertex",
"Remove Zero Area Faces",
"Remove Isolated pieces (wrt Face Num.)",
"Remove Unreferenced Vertex",
"Extract Information"
]
// one out of the list of names you get with /bundles
// always use the "name" field you get from GET /bundles as the name
// might change. You can cache the names locally but be sure to expire
// once in a while.
"template": "basic",
// the bundle name to be used for this job
// in the future its also possible to override templte specific settings and options (shown but noop)
//"bundle": {
// "name": "modelconvert.bundles.pop", // this can also contain a bundle spec and related data
// "settings": {
// "aopt.pop": true,
// "aopt.command": "{command} {input} {output} -what -ever={0} -is -required",
// "meshlab.enabled": false,
// }
// }
}
For exmaple a simple payload to convert a single model without meshalb
sourced from a URL could look like this::
{
"payload": "http://domain.tld/model.obj",
"template": "basic"
}
In return you will get a json response with various data about
your request::
{
// clear text informational message, HTTP status code serves as numeric indicator
"message": "Job accepted with ID 123",
// the task ID the job is running on
"task_id": "123",
// poll URI for checking less frequently for results
"job_url": "full.host/v1/jobs/123",
// URI for status updates through push protocl. This implements
// the W3C EventSource specification. So your client needs to
// support this in order to reciece push updates.
"progress_url": "full.host/v1/stream/123",
}
"""
options = dict() # options passted to task
data = request.json
if not data or not 'payload' in data:
resp = jsonify(message="No payload provided. You need to specify what to convert.")
resp.status_code = 400 # bad resquest
return resp
url = urlparse.urlparse(data['payload'])
current_app.logger.info("Found {0} payload".format(url.scheme))
if url.scheme == 'http':
#
# FIXME BIGTIME
# THE URL DOWNLOADING SHOULD OCCUR IN THE TASK AND NOT BLOCK
# THE INTERFACE
#
if not security.is_allowed_host(data['payload']):
resp = jsonify(message="Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url.netloc, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])))
resp.status_code = 403 #forbidden
return resp
# download file to disk
r = requests.get(data['payload'], stream=True, verify=False)
filename = werkzeug.secure_filename(os.path.split(data['payload'])[-1].split("?")[0])
# FIXME: this should check the mimetype in the http response header
# as well
if not security.is_allowed_file(filename):
resp = jsonify(message="Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']))
resp.status_code = 403 #forbidden
return resp
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
resp = jsonify(message="File too big. Please don't try to use files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])))
resp.status_code = 416 # request range unsatifieable
return resp
else:
# create a UUID like hash for temporary file storage
hash = uuid.uuid4().hex
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, filename)
with open(filename, "wb") as fdata:
fdata.write(r.content)
options.update(hash=hash)
else:
resp = jsonify(message="Could not download file {0} Status code: {1}".format(data['payload'], r.status_code))
resp.status_code = 416 # request range unsatifieable
return resp
elif url.scheme == 'bucket':
if not 'payload_filename' in data:
resp = jsonify(message="Please specify the payload_filename attribute in your request.")
resp.status_code = 400 # bad
return resp
current_app.logger.debug("Using Bucket ID: {0} with entry point filename".format(url.netloc, filename))
options.update(hash=url.netloc)
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], url.netloc)
filename = os.path.join(upload_directory, data['payload_filename'])
def upload():
logger = current_app.logger
"""
The upload method takes a uploaded file and puts it into
the celery processing queue using the Redis backend. Filename
integrity is enforced by renaming the uploaded file to a unique
name and deleting it after successfull processing.
"""
if request.method == 'POST':
# FIXME: convert this to WTForms
meshlab = request.form.getlist('meshlab')
template = request.form['template']
if 'email_to' in request.form:
email_to = request.form['email_to']
else:
email_to=None
file = request.files['file']
metadata = request.files['metadata']
url = request.form['url']
# options to pass to convertion task
options = dict()
# create a UUID like hash for temporary file storage
hash = uuid.uuid4().hex
options.update(hash=hash)
# This whole section is kind of flimsy.
# - code more idiomatic
# - download should be performed asynchrounously, though some checks
# should be performed here (allowed hosts, filenames, etc.)
# - downloading a url triggered via public web form is a HUGE security
# risk. Basically anyone can kill our sever by pasting a url to be
# downloaded. Therefore, at the moment, the ULRs are restricted to
# the ALLOWED_DOWNLOAD_HOSTS settings.
# error handling can be done smarter
# URL instead of file
if url:
# basic security
if not security.is_allowed_host(url):
flash("Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])), 'error')
return render_template('frontend/index.html')
# download file to disk
r = requests.get(url, stream=True, verify=False)
filename = secure_filename(os.path.split(url)[-1].split("?")[0])
# FIXME: this should check the mimetype in the http response header
# as well
if not security.is_allowed_file(filename):
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
flash("File too big. Please don't upload files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])), 'error')
return render_template('frontend/index.html')
else:
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, filename)
with open(filename, "wb") as data:
data.write(r.content)
else:
flash("Could not download file {0} Status code: {1}".format(url, r.status_code), 'error')
return render_template('frontend/index.html')
else:
# in case of file upload place the uploaded file in a folder
# named <uuid>
if file and security.is_allowed_file(file.filename):
filename = secure_filename(file.filename)
upload_directory = os.path.join(current_app.config['UPLOAD_PATH'], hash)
os.mkdir(upload_directory)
filename = os.path.join(upload_directory, file.filename)
file.save(filename)
# in case the user uploaded a meta file, store this as well
# FIXME make sure only processed when valid template selection
if metadata and not compression.is_archive(filename):
meta_filename = os.path.join(upload_directory, 'metadata' + os.path.splitext(metadata.filename)[1])
metadata.save(meta_filename)
# options for task
options.update(meta_filename=meta_filename)
else:
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if email_to:
# we need to add at least captcha system to protect from
# spammers, for now setting the sender env var enables the
# email system, use with care behind pw protected
if current_app.config['DEFAULT_MAIL_SENDER'] == 'noreply@localhost':
options.update(email_to=None)
else:
options.update(email_to=email_to)
if meshlab:
options.update(meshlab=meshlab)
options.update(
template=template
)
retval = tasks.convert_model.apply_async((filename, options))
return redirect(url_for('frontend.status', task_id=retval.task_id))
return render_template('frontend/index.html')
def upload():
"""
The upload method takes a uploaded file and puts it into
the celery processing queue using the Redis backend. Filename
integrity is enforced by renaming the uploaded file to a unique
name and deleting it after successfull processing.
"""
if request.method == 'POST':
# FIXME: convert this to WTForms
meshlab = request.form.getlist('meshlab')
aopt = request.form['aopt']
template = request.form['template']
file = request.files['file']
url = request.form['url']
# options to pass to convertion task
options = dict()
# create a UUID like hash for temporary file storage
#hash = "%032x" % random.getrandbits(128)
hash = uuid.uuid4().hex
options.update(hash=hash)
# This whole section is kind of flimsy.
# - download should be performed asynchrounously, though some checks
# should be performed here (allowed hosts, filenames, etc.)
# - downloading a url triggered via public web form is a HUGE security
# risk. Basically anyone can kill our sever by pasting a url to be
# downloaded. Therefore, at the moment, the ULRs are restricted to
# the ALLOWED_DOWNLOAD_HOSTS settings.
# error handling can be done smarter
# URL instead of file
if url:
# basic security
if not security.is_allowed_host(url):
flash("Tried to download from a insecure source ({0}). Only the following hosts are allowed: {1}".format(url, ", ".join(current_app.config['ALLOWED_DOWNLOAD_HOSTS'])), 'error')
return render_template('frontend/index.html')
# download file to disk
r = requests.get(url, stream=True, verify=False)
filename = secure_filename(os.path.split(url)[-1].split("?")[0])
filename = os.path.join(current_app.config['UPLOAD_PATH'], filename)
if not security.is_allowed_file(filename):
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if r.status_code == requests.codes.ok:
if int(r.headers['content-length']) > current_app.config['MAX_CONTENT_LENGTH']:
flash("File too big. Please don't upload files greater than {0}".format(humanize.bytes(current_app.config['MAX_CONTENT_LENGTH'])), 'error')
return render_template('frontend/index.html')
else:
with open(filename, "wb") as data:
data.write(r.content)
else:
flash("Could not download file {0} Status code: {1}".format(url, r.status_code), 'error')
return render_template('frontend/index.html')
else:
# in case of file upload
if file and security.is_allowed_file(file.filename):
filename = secure_filename(file.filename)
# anonymize filename, keep extension and save
filename = os.path.join(current_app.config['UPLOAD_PATH'],
hash + os.path.splitext(file.filename)[1])
file.save(filename)
else:
flash("Please upload a file of the following type: %s" %
", ".join(current_app.config['ALLOWED_EXTENSIONS']), 'error')
return render_template('frontend/index.html')
if meshlab:
options.update(meshlab=meshlab)
# in case the user uploaded a meta file, store this as well
# FIXME make sure only processed when valid template selection
# (DoS)
metadata = request.files['metadata']
if metadata:
meta_filename = os.path.join(current_app.config['UPLOAD_PATH'], hash, 'metadata' + os.path.splitext(metadata.filename)[1])
metadata.save(meta_filename)
options.update(meta_filename=meta_filename)
options.update(
aopt=aopt,
template=template
)
retval = tasks.convert_model.apply_async((filename, options))
return redirect(url_for('frontend.status', task_id=retval.task_id))
return render_template('frontend/index.html')
