def add_next_exec_info(pipe_id, pipe):
assert uv_url
assert uv_api_url
assert pipe_id
assert pipe
error_msg = None
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
exec_or_schedule_id, next_exec, next_exec_status, after_pipelines = uv_api.get_next_execution_info(pipe_id)
pipe['next_exec'] = format_date(next_exec)
pipe['next_exec_status'] = STATUSES[next_exec_status]
if next_exec_status and exec_or_schedule_id: # execution is queued or running
pipe['next_exec_sched_url'] = '{0}/#!ExecutionList/exec={1}'.format(uv_url, exec_or_schedule_id)
else: # is only scheduled
pipe['next_exec_sched_url'] = '{0}/#!Scheduler'.format(uv_url) # TODO link to schedule
if after_pipelines:
pipe['next_exec'] = _(u"After pipeline(s) with name: '{0}'").format(u"', '".join(after_pipelines))
return
except urllib2.HTTPError, e:
error_msg =_("Couldn't get pipeline next execution information: {error}")\
.format(error=e.msg)
def get_pipeline(pipe_id):
assert uv_api_url
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
pipe = uv_api.get_pipeline_by_id(pipe_id)
return pipe, []
except urllib2.HTTPError, e:
error_msg =_("Couldn't retrieve pipeline information: {error}").format(error=e.msg)
return None, [error_msg]
def disable_schedules_for_pipe(pipe_id):
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
schedules = uv_api.get_all_schedules(pipe_id)
for schedule in schedules:
if not schedule.get('enabled', True):
continue # already disabled
schedule['enabled'] = False
uv_api.edit_pipe_schedule(pipe_id, schedule)
except Exception, e:
err_msg = e.msg or ""
log.error('Failed to disable schedules for pipeline id {0}: {1}'.format(pipe_id, err_msg.encode("utf-8")))
def disable_schedules_for_pipe(pipe_id):
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
schedules = uv_api.get_all_schedules(pipe_id)
for schedule in schedules:
if not schedule.get('enabled', True):
continue # already disabled
schedule['enabled'] = False
uv_api.edit_pipe_schedule(pipe_id, schedule)
except Exception, e:
err_msg = e.msg or ""
log.error(
'Failed to disable schedules for pipeline id {0}: {1}'.format(
pipe_id, err_msg.encode("utf-8")))
def execute_now(self, id, pipeline_id):
err_msg = None
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
# get dataset org id
self._load(id)
actor_id = session.get('ckanext-cas-actorid', None)
# get id of logged user
user_id = None
if c.userobj:
user_id = c.userobj.id
execution = uv_api.execute_now(pipeline_id, user_id=user_id, user_actor_id=actor_id)
log.debug("started execution: {0}".format(execution))
except Exception, e:
err_msg = _(u"Couldn't execute pipeline: {error}").format(error=e.msg)
log.exception(e)
def add_last_exec_info(pipe_id, pipe):
assert uv_url
assert uv_api_url
assert pipe_id
assert pipe
error_msg = None
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
last_exec = uv_api.get_last_finished_execution(pipe_id)
if not last_exec:
return
# adding it to pipe
pipe['last_exec'] = format_date(last_exec['start'])
pipe['last_exec_status'] = STATUSES[last_exec['status']]
pipe['last_exec_link'] = '{0}/#!ExecutionList/exec={1}'\
.format(uv_url, last_exec['id'])
except urllib2.HTTPError, e:
error_msg =_("Couldn't get pipeline last execution information: {error}")\
.format(error=e.msg)
def execute_now(self, id, pipeline_id):
err_msg = None
try:
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
# get dataset org id
self._load(id)
actor_id = session.get('ckanext-cas-actorid', None)
# get id of logged user
user_id = None
if c.userobj:
user_id = c.userobj.id
execution = uv_api.execute_now(pipeline_id,
user_id=user_id,
user_actor_id=actor_id)
log.debug("started execution: {0}".format(execution))
except Exception, e:
err_msg = _(u"Couldn't execute pipeline: {error}").format(
error=e.msg)
log.exception(e)
def get_all_pipelines(only_visible=False):
assert uv_api_url
try:
if c.userobj:
user_external_id = c.userobj.id
else:
# raise error
err_msg = _('Error: Only logged in user can associate pipelines.')
log.error(err_msg)
h.flash_error(err_msg)
return []
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
if only_visible:
pipes = uv_api.get_visible_pipelines(user_external_id)
else:
pipes = uv_api.get_pipelines(user_external_id)
return pipes
except urllib2.HTTPError, e:
error_msg =_("Couldn't retrieve information about pipelines: {error}").format(error=e.msg)
h.flash_error(error_msg)
log.error(e)
return None
def create_pipe_manually(self, id):
assert uv_url
data = request.POST
name = ''
description = ''
if u'name' in data:
name = data[u'name']
if u'description' in data:
description = data[u'description']
if not name.strip():
h.flash_error(_(u"Pipeline name is required."))
self._load(id)
return render('pipeline/create_pipeline.html',
extra_vars={'descr': description,
'form_action': 'create_pipe_manually'})
data_dict = {'id': id}
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj}
assert uv_api_url
open_on_load = None
msg = None
err_msg = None
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# creating new Pipe in UV
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
actor_id = session.get('ckanext-cas-actorid', None)
user_id = None
if c.userobj:
user_id = c.userobj.id
new_pipe = uv_api.create_pipeline(name, description, user_id, actor_id)
# associate it with dataset
if not new_pipe:
err_msg = _(u"Pipeline wasn't created.")
elif Pipelines.by_pipeline_id(new_pipe['id']): # checks if already exists
err_msg = _(u'Pipeline is already associated to some dataset.')
else:
# adds pipe association
package_pipe = Pipelines(package['id'], new_pipe['id'], name=new_pipe['name'])
package_pipe.save() # this adds and commits too
msg = _(u"Pipeline {name} assigned successfully.").format(name=new_pipe['name'])
open_on_load = uv_url + '/#!PipelineEdit/{pipe_id}'.format(pipe_id=new_pipe['id'])
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(401, _(u'User {user} not authorized to edit {id}').format(user=c.user, id=id))
except Exception, e:
log.exception(e)
err_msg = _(u"Couldn't create/associate pipeline: {error}").format(error=e.msg)
self._load(id)
vars = {
'form_action': 'create_pipe_manually',
'name': name,
'descr': description,
'err_msg': err_msg
}
return render('pipeline/create_pipeline.html', extra_vars = vars)
def create_pipe_manually(self, id):
assert uv_url
data = request.POST
name = ''
description = ''
if u'name' in data:
name = data[u'name']
if u'description' in data:
description = data[u'description']
if not name.strip():
h.flash_error(_(u"Pipeline name is required."))
self._load(id)
return render('pipeline/create_pipeline.html',
extra_vars={
'descr': description,
'form_action': 'create_pipe_manually'
})
data_dict = {'id': id}
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
assert uv_api_url
open_on_load = None
msg = None
err_msg = None
try:
check_access('package_update', context, data_dict)
package = get_action('package_show')(context, data_dict)
# creating new Pipe in UV
uv_api = UVRestAPIWrapper(uv_api_url, uv_api_auth)
actor_id = session.get('ckanext-cas-actorid', None)
user_id = None
if c.userobj:
user_id = c.userobj.id
new_pipe = uv_api.create_pipeline(name, description, user_id,
actor_id)
# associate it with dataset
if not new_pipe:
err_msg = _(u"Pipeline wasn't created.")
elif Pipelines.by_pipeline_id(
new_pipe['id']): # checks if already exists
err_msg = _(u'Pipeline is already associated to some dataset.')
else:
# adds pipe association
package_pipe = Pipelines(package['id'],
new_pipe['id'],
name=new_pipe['name'])
package_pipe.save() # this adds and commits too
msg = _(u"Pipeline {name} assigned successfully.").format(
name=new_pipe['name'])
open_on_load = uv_url + '/#!PipelineEdit/{pipe_id}'.format(
pipe_id=new_pipe['id'])
except NotFound:
abort(404, _(u'Dataset not found'))
except NotAuthorized:
abort(
401,
_(u'User {user} not authorized to edit {id}').format(
user=c.user, id=id))
except Exception, e:
log.exception(e)
err_msg = _(u"Couldn't create/associate pipeline: {error}").format(
error=e.msg)
self._load(id)
vars = {
'form_action': 'create_pipe_manually',
'name': name,
'descr': description,
'err_msg': err_msg
}
return render('pipeline/create_pipeline.html', extra_vars=vars)