def _validate(self, **kw):
debug('_validate %s' % kw,)
bs_private = copy.deepcopy(json.loads(kw['bs_private']))
plugin_id = bs_private['pp']['id']
plug = operations.get_plugin_byId(plugin_id)
obj = plug
info = obj.info
form = info.get('output')()
# prefilling form (if the validation fail, respons must contain the form prefilled
# perhaps we can do this only if the form validation fail)
if 'prefill' in bs_private:
prefill_fields(info.get('in'), form, bs_private['prefill'], kw, replace_value=False)
debug('prefill in validation',)
del bs_private['prefill']
response.headers['Access-Control-Allow-Headers'] = 'X-CSRF-Token'
response.headers['Access-Control-Allow-Origin'] = '*'
form = form().req()
try:
kw = form.validate(kw)
except twc.ValidationError as e:
main_proxy = tg.config.get('main.proxy')
e.widget.action = main_proxy + tg.url('plugins/fetch', {'oid': plugin_id})
util.print_traceback()
return {'page': 'plugin', 'desc': info.get('description'), 'title': info.get('title'), 'widget': e.widget}
return json.dumps({'validated': True, 'params': kw})
def _validate(self, **kw):
debug('_validate %s' % kw, )
bs_private = copy.deepcopy(json.loads(kw['bs_private']))
plugin_id = bs_private['pp']['id']
plug = operations.get_plugin_byId(plugin_id)
obj = plug
info = obj.info
form = info.get('output')()
# prefilling form (if the validation fail, respons must contain the form prefilled
# perhaps we can do this only if the form validation fail)
if 'prefill' in bs_private:
prefill_fields(info.get('in'),
form,
bs_private['prefill'],
kw,
replace_value=False)
debug('prefill in validation', )
del bs_private['prefill']
response.headers['Access-Control-Allow-Headers'] = 'X-CSRF-Token'
response.headers['Access-Control-Allow-Origin'] = '*'
form = form().req()
try:
kw = form.validate(kw)
except twc.ValidationError as e:
main_proxy = tg.config.get('main.proxy')
e.widget.action = main_proxy + tg.url('plugins/fetch',
{'oid': plugin_id})
util.print_traceback()
return {
'page': 'plugin',
'desc': info.get('description'),
'title': info.get('title'),
'widget': e.widget
}
return json.dumps({'validated': True, 'params': kw})
def shutilerror(func, path, einfo):
print '[shutil error] with %s. On path %s' % (func, path)
util.print_traceback()
def plugin_job(user, plug, inputs_directory, outputs_directory, dwdfiles, plugin_info,
user_parameters, service_callback, bioscript_callback, **form_parameters):
debug('FETCHING FILES ... %s' % form_parameters)
raised = False
task_id = plugin_job.request.id
try:
# FETCHING FILES
current_task.update_state(state='FETCHING FILES')
inputs_directory = fetchurls(user, plug, dwdfiles, inputs_directory, form_parameters)
debug('PLUGIN PROCESS STARTED ...')
# PLUGIN PROCESS
current_task.update_state(state='STARTED')
try:
debug('user params : %s' % user_parameters)
user_parameters = json.loads(user_parameters)
except TypeError as e:
debug(e)
if service_callback is not None:
callback_service(service_callback, plugin_info['generated_id'], task_id, 'RUNNING', additional=user_parameters)
debug('task launched user.name: %s, indir: %s, oudir: %s' % (user.name, inputs_directory, outputs_directory))
# get plugin class
plugin = operations.get_plugin_byId(plugin_info['generated_id'])
if plugin is None:
raise Exception('Plugin not found by the worker.')
plugin.is_debug = DEBUG_PLUGINS
debug('plugin operation start %s' % form_parameters)
plugin._start_timer()
results = []
# call plugin with form parameters
try:
os.chdir(TMP_DIR)
ret = plugin(**form_parameters)
results = [{'is_file': False,
'value': ret}]
except Exception as e:
debug("ERROR %s" % e)
util.print_traceback()
if service_callback is not None:
user_parameters.update({'error': str(e)})
callback_service(service_callback, plugin_info['generated_id'], task_id, 'FAILED', additional=user_parameters)
# deleting plugin temporary files
for todel in plugin.tmp_files:
debug('deleting %s' % todel)
shutil.rmtree(todel, onerror=shutilerror)
#debug('deleting %s' % outputs_directory)
#shutil.rmtree(outputs_directory, onerror=shutilerror)
if file_is_in_bs(TMP_DIR, inputs_directory):
debug('deleting %s' % inputs_directory)
shutil.rmtree(inputs_directory, onerror=shutilerror)
if len(plugin.debug_stack) > 0:
debug("Adding debug stack to error")
debug_error = '\n [x] DEBUG STACK : %s' % '\n'.join([d for d in plugin.debug_stack])
if not e.args:
e.args = ('',)
e.args = (e.args[0] + debug_error,) + e.args[1:]
raised = True
raise
# mkdir output directory
out = os.path.join(outputs_directory, task_id)
debug('mkdir out path : %s' % out)
try:
os.mkdir(out)
except OSError, e:
if e.errno == errno.EEXIST:
io.rm(out)
os.mkdir(out)
debug('moving files')
# moving files to the output directory
for output_file, output_type in plugin.output_files:
debug('f: %s to : %s' % (output_file, out))
out_path = os.path.join(out, os.path.split(output_file)[1])
io.mv(output_file, out)
results.append({'is_file': True, 'path': out_path, 'type': output_type})
# deleting plugin temporary files
for todel in plugin.tmp_files:
debug('deleting %s' % todel)
shutil.rmtree(todel, onerror=shutilerror)
if file_is_in_bs(TMP_DIR, inputs_directory):
debug('deleting %s' % inputs_directory)
shutil.rmtree(inputs_directory, onerror=shutilerror)
# updating bioscript with the results
debug('CALLBACK to %s for task %s' % (bioscript_callback, task_id))
debug(cbbioscript(bioscript_callback, task_id, json.dumps(results)))
debug("END")
# callback
if service_callback is not None:
callback_service(service_callback, plugin_info['generated_id'], task_id, 'SUCCESS',
results=json.dumps(results), additional=user_parameters)
def validate(self, **kw):
"""
plugin parameters validation
"""
user = util.get_user(tg.request)
# get private bioscript parameters
if not 'bs_private' in kw:
debug('bs_private not found')
tg.abort(400, "Plugin identifier not found in the request.")
bs_private = copy.deepcopy(json.loads(kw['bs_private']))
debug('\n[plugin controller]\nVALIDATE %s' % kw,)
# get the plugin from the private parameters
plugin_id = 0
try:
plugin_id = bs_private['pp']['id']
except KeyError:
tg.abort(400, "Plugin identifier not found in the request.")
if plugin_id == 0:
tg.abort(400, "Plugin identifier not found in the request.")
plug = operations.get_plugin_byId(plugin_id)
if plug is None:
tg.abort(400, "Bad plugin identifier.")
# get the plugin from database
plugin_db = DBSession.query(Plugin).filter(Plugin.generated_id == plug.unique_id()).first()
# validate the form
# we must do an HTTP call because I don't find a way
# to call the 'render' method on 'plugin_validate' template
request_url = tg.config.get('main.proxy') + '/plugins/_validate'
validated = True
info = plug.info
# get the callback if any
callback = kw.get('callback', 'callback')
try:
form = urllib2.urlopen(request_url, urllib.urlencode(kw)).read()
# add some header because the request can come from another domain
response.headers['Access-Control-Allow-Headers'] = 'X-CSRF-Token'
response.headers['Access-Control-Allow-Origin'] = '*'
try:
form = json.loads(form)
except:
pass
validated = isinstance(form, dict)
except Exception as e:
util.print_traceback()
validated = False
form = 'Problem with Bioscript server.'
if not validated:
debug('Validation failed',)
return json.dumps({'validation': 'failed',
'desc': info.get('description'),
'title': info.get('title'),
'widget': form,
'callback': callback})
# validation passes
new_params = form['params']
new_params = kw
debug('Validation passes with params : %s' % new_params,)
# we regoup all multi stuff in a single list:
# for instance a multi field will give parameters like:
# {SigMulti:1:signals: val1, SigMulti:2:signals: val2, ...}
# and we will transform it to
# {SigMulti: { signals : [val1, val2], ...}
grouped_params = {}
todel = []
for k, v in new_params.iteritems():
m = multipattern.match(k)
if m is not None:
todel.append(k)
if v or isinstance(v, cgi.FieldStorage):
key1, n, key2 = m.groups()
if key1 not in grouped_params:
grouped_params[key1] = {}
if key2 in grouped_params[key1]:
grouped_params[key1][key2][int(n) - 1] = v
elif not key2.endswith('bs_group'):
sl = SparseList()
sl[int(n) - 1] = v
grouped_params[key1][key2] = sl
debug('group "multi" params: %s' % grouped_params)
# debug('check fieldstorages',)
# # must keep fieldstorages because they were converted to str
# fs_bk = []
# import cgi
# for k, v in kw.iteritems():
# if isinstance(v, cgi.FieldStorage):
# fs_bk.append((k, v))
# debug(fs_bk)
# for fsk, fsv in fs_bk:
# m = multipattern.match(fsk)
# if m:
# key1, n, key2 = m.groups()
# grouped_params[key1][key2][int(n) - 1] = fsv
# else:
# grouped_params[fsk] = fsv
new_params.update(grouped_params)
# delete multi parameters
for td in todel:
del new_params[td]
# but we need to keep all params that are multi and urls
kw = new_params
#remove private parameters from the request
if 'bs_private' in kw:
del kw['bs_private']
if 'key' in kw:
del kw['key']
debug('New params are : %s' % new_params,)
# update plugin arameters
# log the request, it's a valid one
plugin_request = _log_form_request(plugin_id=plugin_db.id, user=user, parameters=dict(kw))
DBSession.add(plugin_request)
debug('get output directory',)
# get output directory to write results
outputs_directory = filemanager.temporary_directory(constants.paths['data'])
service_callback = None
# if the user is a service, get parameters from configuration
if user.is_service:
debug('is service',)
o = services.service_manager.get(user.name, constants.SERVICE_RESULT_ROOT_PARAMETER)
if o:
outputs_directory = o
service_callback = services.service_manager.get(user.name, constants.SERVICE_CALLBACK_URL_PARAMETER)
debug('Write result in %s' % outputs_directory,)
# get prvate parameters from the request
private_parameters = bs_private.get('app', "{}")
# get response configuration from the request
resp_config = bs_private.get('cfg', None)
plugin_info = {'title': info['title'],
'plugin_id': plugin_db.id,
'generated_id': plugin_db.generated_id,
'description': info['description'],
'path': info['path'],
'in': info['in'],
'out': info['out'],
'meta': info['meta']}
# define the bioscript callback
bioscript_callback = tg.config.get('main.proxy') + '/' + tg.url('plugins/callback_results')
debug("callback on bs : %s " % bioscript_callback,)
# if some files come from a file field, we must download them directly
inputs_directory, dwdfiles = filemanager.fetchfilefields(user, plug, kw)
# chain jobs : fetch files then plugin process
async_res = tasks.plugin_job.delay(user,
plug,
inputs_directory,
outputs_directory,
dwdfiles,
plugin_info,
private_parameters,
service_callback,
bioscript_callback,
**kw)
task_id = async_res.task_id
debug('Launch task %s' % task_id,)
# log the job request
_log_job_request(plugin_request.id, task_id)
#prepare the response
resp = {'validation': 'success',
'plugin_id': plugin_id,
'task_id': task_id,
'callback': callback,
'app': private_parameters}
if resp_config and resp_config.get('plugin_info', '') == 'min':
resp.update({'plugin_info': json.dumps({'title': info['title'],
'description': info['description'],
'path': info['path'],
'in': info['in'],
'out': info['out'],
'meta': info['meta']})})
return json.dumps(resp)
def shutilerror(func, path, einfo):
print '[shutil error] with %s. On path %s' % (func, path)
util.print_traceback()
def plugin_job(user, plug, inputs_directory, outputs_directory, dwdfiles,
plugin_info, user_parameters, service_callback,
bioscript_callback, **form_parameters):
debug('FETCHING FILES ... %s' % form_parameters)
raised = False
task_id = plugin_job.request.id
try:
# FETCHING FILES
current_task.update_state(state='FETCHING FILES')
inputs_directory = fetchurls(user, plug, dwdfiles, inputs_directory,
form_parameters)
debug('PLUGIN PROCESS STARTED ...')
# PLUGIN PROCESS
current_task.update_state(state='STARTED')
try:
debug('user params : %s' % user_parameters)
user_parameters = json.loads(user_parameters)
except TypeError as e:
debug(e)
if service_callback is not None:
callback_service(service_callback,
plugin_info['generated_id'],
task_id,
'RUNNING',
additional=user_parameters)
debug('task launched user.name: %s, indir: %s, oudir: %s' %
(user.name, inputs_directory, outputs_directory))
# get plugin class
plugin = operations.get_plugin_byId(plugin_info['generated_id'])
if plugin is None:
raise Exception('Plugin not found by the worker.')
plugin.is_debug = DEBUG_PLUGINS
debug('plugin operation start %s' % form_parameters)
plugin._start_timer()
results = []
# call plugin with form parameters
try:
os.chdir(TMP_DIR)
ret = plugin(**form_parameters)
results = [{'is_file': False, 'value': ret}]
except Exception as e:
debug("ERROR %s" % e)
util.print_traceback()
if service_callback is not None:
user_parameters.update({'error': str(e)})
callback_service(service_callback,
plugin_info['generated_id'],
task_id,
'FAILED',
additional=user_parameters)
# deleting plugin temporary files
for todel in plugin.tmp_files:
debug('deleting %s' % todel)
shutil.rmtree(todel, onerror=shutilerror)
#debug('deleting %s' % outputs_directory)
#shutil.rmtree(outputs_directory, onerror=shutilerror)
if file_is_in_bs(TMP_DIR, inputs_directory):
debug('deleting %s' % inputs_directory)
shutil.rmtree(inputs_directory, onerror=shutilerror)
if len(plugin.debug_stack) > 0:
debug("Adding debug stack to error")
debug_error = '\n [x] DEBUG STACK : %s' % '\n'.join(
[d for d in plugin.debug_stack])
if not e.args:
e.args = ('', )
e.args = (e.args[0] + debug_error, ) + e.args[1:]
raised = True
raise
# mkdir output directory
out = os.path.join(outputs_directory, task_id)
debug('mkdir out path : %s' % out)
try:
os.mkdir(out)
except OSError, e:
if e.errno == errno.EEXIST:
io.rm(out)
os.mkdir(out)
debug('moving files')
# moving files to the output directory
for output_file, output_type in plugin.output_files:
debug('f: %s to : %s' % (output_file, out))
out_path = os.path.join(out, os.path.split(output_file)[1])
io.mv(output_file, out)
results.append({
'is_file': True,
'path': out_path,
'type': output_type
})
# deleting plugin temporary files
for todel in plugin.tmp_files:
debug('deleting %s' % todel)
shutil.rmtree(todel, onerror=shutilerror)
if file_is_in_bs(TMP_DIR, inputs_directory):
debug('deleting %s' % inputs_directory)
shutil.rmtree(inputs_directory, onerror=shutilerror)
# updating bioscript with the results
debug('CALLBACK to %s for task %s' % (bioscript_callback, task_id))
debug(cbbioscript(bioscript_callback, task_id, json.dumps(results)))
debug("END")
# callback
if service_callback is not None:
callback_service(service_callback,
plugin_info['generated_id'],
task_id,
'SUCCESS',
results=json.dumps(results),
additional=user_parameters)
def validate(self, **kw):
"""
plugin parameters validation
"""
user = util.get_user(tg.request)
# get private bioscript parameters
if not 'bs_private' in kw:
debug('bs_private not found')
tg.abort(400, "Plugin identifier not found in the request.")
bs_private = copy.deepcopy(json.loads(kw['bs_private']))
debug('\n[plugin controller]\nVALIDATE %s' % kw, )
# get the plugin from the private parameters
plugin_id = 0
try:
plugin_id = bs_private['pp']['id']
except KeyError:
tg.abort(400, "Plugin identifier not found in the request.")
if plugin_id == 0:
tg.abort(400, "Plugin identifier not found in the request.")
plug = operations.get_plugin_byId(plugin_id)
if plug is None:
tg.abort(400, "Bad plugin identifier.")
# get the plugin from database
plugin_db = DBSession.query(Plugin).filter(
Plugin.generated_id == plug.unique_id()).first()
# validate the form
# we must do an HTTP call because I don't find a way
# to call the 'render' method on 'plugin_validate' template
request_url = tg.config.get('main.proxy') + '/plugins/_validate'
validated = True
info = plug.info
# get the callback if any
callback = kw.get('callback', 'callback')
try:
form = urllib2.urlopen(request_url, urllib.urlencode(kw)).read()
# add some header because the request can come from another domain
response.headers['Access-Control-Allow-Headers'] = 'X-CSRF-Token'
response.headers['Access-Control-Allow-Origin'] = '*'
try:
form = json.loads(form)
except:
pass
validated = isinstance(form, dict)
except Exception as e:
util.print_traceback()
validated = False
form = 'Problem with Bioscript server.'
if not validated:
debug('Validation failed', )
return json.dumps({
'validation': 'failed',
'desc': info.get('description'),
'title': info.get('title'),
'widget': form,
'callback': callback
})
# validation passes
new_params = form['params']
new_params = kw
debug('Validation passes with params : %s' % new_params, )
# we regoup all multi stuff in a single list:
# for instance a multi field will give parameters like:
# {SigMulti:1:signals: val1, SigMulti:2:signals: val2, ...}
# and we will transform it to
# {SigMulti: { signals : [val1, val2], ...}
grouped_params = {}
todel = []
for k, v in new_params.iteritems():
m = multipattern.match(k)
if m is not None:
todel.append(k)
if v or isinstance(v, cgi.FieldStorage):
key1, n, key2 = m.groups()
if key1 not in grouped_params:
grouped_params[key1] = {}
if key2 in grouped_params[key1]:
grouped_params[key1][key2][int(n) - 1] = v
elif not key2.endswith('bs_group'):
sl = SparseList()
sl[int(n) - 1] = v
grouped_params[key1][key2] = sl
debug('group "multi" params: %s' % grouped_params)
# debug('check fieldstorages',)
# # must keep fieldstorages because they were converted to str
# fs_bk = []
# import cgi
# for k, v in kw.iteritems():
# if isinstance(v, cgi.FieldStorage):
# fs_bk.append((k, v))
# debug(fs_bk)
# for fsk, fsv in fs_bk:
# m = multipattern.match(fsk)
# if m:
# key1, n, key2 = m.groups()
# grouped_params[key1][key2][int(n) - 1] = fsv
# else:
# grouped_params[fsk] = fsv
new_params.update(grouped_params)
# delete multi parameters
for td in todel:
del new_params[td]
# but we need to keep all params that are multi and urls
kw = new_params
#remove private parameters from the request
if 'bs_private' in kw:
del kw['bs_private']
if 'key' in kw:
del kw['key']
debug('New params are : %s' % new_params, )
# update plugin arameters
# log the request, it's a valid one
plugin_request = _log_form_request(plugin_id=plugin_db.id,
user=user,
parameters=dict(kw))
DBSession.add(plugin_request)
debug('get output directory', )
# get output directory to write results
outputs_directory = filemanager.temporary_directory(
constants.paths['data'])
service_callback = None
# if the user is a service, get parameters from configuration
if user.is_service:
debug('is service', )
o = services.service_manager.get(
user.name, constants.SERVICE_RESULT_ROOT_PARAMETER)
if o:
outputs_directory = o
service_callback = services.service_manager.get(
user.name, constants.SERVICE_CALLBACK_URL_PARAMETER)
debug('Write result in %s' % outputs_directory, )
# get prvate parameters from the request
private_parameters = bs_private.get('app', "{}")
# get response configuration from the request
resp_config = bs_private.get('cfg', None)
plugin_info = {
'title': info['title'],
'plugin_id': plugin_db.id,
'generated_id': plugin_db.generated_id,
'description': info['description'],
'path': info['path'],
'in': info['in'],
'out': info['out'],
'meta': info['meta']
}
# define the bioscript callback
bioscript_callback = tg.config.get('main.proxy') + '/' + tg.url(
'plugins/callback_results')
debug("callback on bs : %s " % bioscript_callback, )
# if some files come from a file field, we must download them directly
inputs_directory, dwdfiles = filemanager.fetchfilefields(
user, plug, kw)
# chain jobs : fetch files then plugin process
async_res = tasks.plugin_job.delay(user, plug, inputs_directory,
outputs_directory, dwdfiles,
plugin_info, private_parameters,
service_callback,
bioscript_callback, **kw)
task_id = async_res.task_id
debug('Launch task %s' % task_id, )
# log the job request
_log_job_request(plugin_request.id, task_id)
#prepare the response
resp = {
'validation': 'success',
'plugin_id': plugin_id,
'task_id': task_id,
'callback': callback,
'app': private_parameters
}
if resp_config and resp_config.get('plugin_info', '') == 'min':
resp.update({
'plugin_info':
json.dumps({
'title': info['title'],
'description': info['description'],
'path': info['path'],
'in': info['in'],
'out': info['out'],
'meta': info['meta']
})
})
return json.dumps(resp)
