def remove_pipe(self, id, pipeline_id):
assert id
assert pipeline_id
try:
data_dict = {'id': id}
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj}
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# id is most probably is package.name so we have to get actual id
pipe = Pipelines(package['id'], pipeline_id).get()
if not pipe:
h.flash_error(_(u"Couldn't remove pipeline, because there is no such pipeline assigned to this dataset."))
base.redirect(h.url_for('pipe_assign', id=id))
else:
pipe_id = pipe.pipeline_id
pipe.delete()
pipe.commit()
h.flash_success(_(u'Pipeline removed from dataset successfully'))
disable_schedules_for_pipe(pipe_id)
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(401, _(u'User {user} not authorized to edit {id}').format(user=c.user, id=id))
base.redirect(h.url_for('pipe_assign', id=id))
def assign(self, id):
pipe = None
data = request.POST
if u'pipeline' in data:
pipe_id = data[u'pipeline']
pipe, err_msg = get_pipeline(pipe_id)
data_dict = {'id': id}
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
if not pipe:
h.flash_notice(_(u"No pipeline selected."))
base.redirect(h.url_for('dataset_pipelines', id=id))
return
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# checks if already exists
if Pipelines.by_pipeline_id(pipe['id']):
h.flash_error(
_(u'Pipeline is already associated to some dataset.'))
base.redirect(h.url_for('pipe_assign', id=id))
else:
# adds pipe association
package_pipe = Pipelines(package['id'],
pipe['id'],
name=pipe['name'])
package_pipe.save() # this adds and commits too
h.flash_success(_(u"Pipeline assigned successfully."))
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(
401,
_(u'User {user} not authorized to edit {id}').format(
user=c.user, id=id))
base.redirect(h.url_for('pipe_assign', id=id))
def assign(self, id):
pipe = None
data = request.POST
if u'pipeline' in data:
pipe_id = data[u'pipeline']
pipe, err_msg = get_pipeline(pipe_id)
data_dict = {'id': id}
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj}
if not pipe:
h.flash_notice(_(u"No pipeline selected."))
base.redirect(h.url_for('dataset_pipelines', id=id))
return
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# checks if already exists
if Pipelines.by_pipeline_id(pipe['id']):
h.flash_error(_(u'Pipeline is already associated to some dataset.'))
base.redirect(h.url_for('pipe_assign', id=id))
else:
# adds pipe association
package_pipe = Pipelines(package['id'], pipe['id'], name=pipe['name'])
package_pipe.save() # this adds and commits too
h.flash_success(_(u"Pipeline assigned successfully."))
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(401, _(u'User {user} not authorized to edit {id}').format(user=c.user, id=id))
base.redirect(h.url_for('pipe_assign', id=id))
def internal_api(context, data_dict=None):
check_and_bust('action', data_dict)
check_and_bust('user_id', data_dict)
user_id = data_dict.get('user_id', None)
change_auth_user(context, user_id)
log.debug('internal_api: action = {0}'.format(data_dict['action']))
log.debug('internal_api: user_id = {0}'.format(user_id))
logic.check_access('internal_api', context, data_dict)
action = data_dict['action']
data = data_dict.get('data', {})
if isinstance(data, basestring):
# if upload data is actually a string
data = json.loads(data)
if 'resource_download' in action:
return resource_download(context, data)
# type == 'FILE'
if data_dict.has_key('upload'):
data['upload'] = data_dict['upload']
data['url'] = ''
# any type
if data_dict.has_key('pipeline_id') and data_dict['pipeline_id']:
log.debug('internal_api: pipeline_id = {0}'.format(data_dict['pipeline_id']))
pipeline_id = data_dict['pipeline_id']
dataset_to_pipeline = Pipelines.by_pipeline_id(pipeline_id)
if dataset_to_pipeline:
# converting pipeline_id to 'id' or 'package_id'
if action in ['resource_create']:
data['package_id'] = dataset_to_pipeline.package_id
elif action in ['package_update', 'package_show']:
data['id'] = dataset_to_pipeline.package_id
else:
raise NotFound('No dataset found for pipeline_id = {0}'.format(pipeline_id))
# type == 'RDF'
if data_dict.has_key('type') and data_dict['type'] == 'RDF':
data['url'] = get_rdf_url(data_dict)
return get_action(action)(context, data)
def get_pipelines_not_assigned():
pipes = get_all_pipelines()
if not pipes:
return []
pipelines_assigned = Pipelines.get_all()
# remove already assigned pipelines
pipes_assigned = []
for pipe in pipelines_assigned:
pipes_assigned.append(pipe.pipeline_id)
unassigned = []
for pipe in pipes:
if not pipe['id'] in pipes_assigned:
unassigned.append(pipe)
return unassigned
def get_dataset_pipelines(package_id):
assert package_id
dataset_pipes = Pipelines.by_dataset_id(package_id)
val = []
try:
# UV connection is OK, we can get pipe info from UV
for pipes in dataset_pipes:
pipe, err_msg = get_pipeline(pipes.pipeline_id)
# name synchronization
if pipe and pipes.name != pipe['name']:
synchonize_name(pipe, pipes)
if pipe:
err_last = add_last_exec_info(pipes.pipeline_id, pipe)
err_next = add_next_exec_info(pipes.pipeline_id, pipe)
if err_last:
err_msg.append(err_last)
if err_next:
err_msg.append(err_next)
pipe['error'] = err_msg
if pipe:
val.append(pipe)
else:
val.append({u'id': pipes.pipeline_id, u'name':pipes.name, u'error': err_msg})
except urllib2.URLError:
h.flash_error(_("Couldn't connect to UnifiedViews server."))
# get info only from DB
for pipes in dataset_pipes:
pipe = {u'id': pipes.pipeline_id, u'name':pipes.name}
val.append(pipe)
return val
def remove_pipe(self, id, pipeline_id):
assert id
assert pipeline_id
try:
data_dict = {'id': id}
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# id is most probably is package.name so we have to get actual id
pipe = Pipelines(package['id'], pipeline_id).get()
if not pipe:
h.flash_error(
_(u"Couldn't remove pipeline, because there is no such pipeline assigned to this dataset."
))
base.redirect(h.url_for('pipe_assign', id=id))
else:
pipe_id = pipe.pipeline_id
pipe.delete()
pipe.commit()
h.flash_success(
_(u'Pipeline removed from dataset successfully'))
disable_schedules_for_pipe(pipe_id)
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(
401,
_(u'User {user} not authorized to edit {id}').format(
user=c.user, id=id))
base.redirect(h.url_for('pipe_assign', id=id))
def create_pipe_manually(self, id):
assert uv_url
data = request.POST
name = ''
description = ''
if u'name' in data:
name = data[u'name']
if u'description' in data:
description = data[u'description']
if not name.strip():
h.flash_error(_(u"Pipeline name is required."))
self._load(id)
return render('pipeline/create_pipeline.html',
extra_vars={'descr': description,
'form_action': 'create_pipe_manually'})
data_dict = {'id': id}
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj}
assert uv_api_url
open_on_load = None
msg = None
err_msg = None
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# creating new Pipe in UV
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
actor_id = session.get('ckanext-cas-actorid', None)
user_id = None
if c.userobj:
user_id = c.userobj.id
new_pipe = uv_api.create_pipeline(name, description, user_id, actor_id)
# associate it with dataset
if not new_pipe:
err_msg = _(u"Pipeline wasn't created.")
elif Pipelines.by_pipeline_id(new_pipe['id']): # checks if already exists
err_msg = _(u'Pipeline is already associated to some dataset.')
else:
# adds pipe association
package_pipe = Pipelines(package['id'], new_pipe['id'], name=new_pipe['name'])
package_pipe.save() # this adds and commits too
msg = _(u"Pipeline {name} assigned successfully.").format(name=new_pipe['name'])
open_on_load = uv_url + '/#!PipelineEdit/{pipe_id}'.format(pipe_id=new_pipe['id'])
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(401, _(u'User {user} not authorized to edit {id}').format(user=c.user, id=id))
except Exception, e:
log.exception(e)
err_msg = _(u"Couldn't create/associate pipeline: {error}").format(error=e.msg)
self._load(id)
vars = {
'form_action': 'create_pipe_manually',
'name': name,
'descr': description,
'err_msg': err_msg
}
return render('pipeline/create_pipeline.html', extra_vars = vars)
def create_pipe_manually(self, id):
assert uv_url
data = request.POST
name = ''
description = ''
if u'name' in data:
name = data[u'name']
if u'description' in data:
description = data[u'description']
if not name.strip():
h.flash_error(_(u"Pipeline name is required."))
self._load(id)
return render('pipeline/create_pipeline.html',
extra_vars={
'descr': description,
'form_action': 'create_pipe_manually'
})
data_dict = {'id': id}
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
assert uv_api_url
open_on_load = None
msg = None
err_msg = None
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# creating new Pipe in UV
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
actor_id = session.get('ckanext-cas-actorid', None)
user_id = None
if c.userobj:
user_id = c.userobj.id
new_pipe = uv_api.create_pipeline(name, description, user_id,
actor_id)
# associate it with dataset
if not new_pipe:
err_msg = _(u"Pipeline wasn't created.")
elif Pipelines.by_pipeline_id(
new_pipe['id']): # checks if already exists
err_msg = _(u'Pipeline is already associated to some dataset.')
else:
# adds pipe association
package_pipe = Pipelines(package['id'],
new_pipe['id'],
name=new_pipe['name'])
package_pipe.save() # this adds and commits too
msg = _(u"Pipeline {name} assigned successfully.").format(
name=new_pipe['name'])
open_on_load = uv_url + '/#!PipelineEdit/{pipe_id}'.format(
pipe_id=new_pipe['id'])
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(
401,
_(u'User {user} not authorized to edit {id}').format(
user=c.user, id=id))
except Exception, e:
log.exception(e)
err_msg = _(u"Couldn't create/associate pipeline: {error}").format(
error=e.msg)
self._load(id)
vars = {
'form_action': 'create_pipe_manually',
'name': name,
'descr': description,
'err_msg': err_msg
}
return render('pipeline/create_pipeline.html', extra_vars=vars)