def delete_design(request, design_id):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
skip_trash = 'skip_trash' in request.GET
try:
workflow = _get_design(design_id)
_check_permission(request,
workflow.owner.username,
_("Access denied: delete design %(id)s.") %
{'id': design_id},
allow_root=True)
if skip_trash:
Workflow.objects.destroy(workflow, request.fs)
else:
workflow.delete(skip_trash=False)
except Workflow.DoesNotExist:
raise StructuredException(code="NOT_FOUND",
message=_('Could not find design %s.') %
design_id,
error_code=404)
return render_json({'status': 0})
def workflow_jobsub_actions(request, workflow):
if request.method not in ['GET', 'POST']:
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be GET or POST request.'), error_code=405)
available_actions = OozieDesign.objects.all()
if request.method == 'POST':
form = ImportJobsubDesignForm(data=request.POST, choices=[(action.id, action.name) for action in available_actions])
if form.is_valid():
try:
design = OozieDesign.objects.get(id=form.cleaned_data['jobsub_id'])
action = convert_jobsub_design(design)
action.workflow = workflow
response = {
'status': 0,
'data': {
'node': model_to_dict(action)
}
}
response['data']['node']['child_links'] = []
return HttpResponse(json.dumps(response), mimetype="application/json")
except OozieDesign.DoesNotExist, e:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Job Designer design does not exist.'), data={'exception': str(e)}, error_code=400)
except (Mapreduce.DoesNotExist, Streaming.DoesNotExist, Java.DoesNotExist), e:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Could not convert Job Designer design.'), data={'exception': str(e)}, error_code=400)
except Exception, e:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error importing node from Job Designer'), data={'exception': str(e)}, error_code=400)
def wrapper(*args, **kwargs):
superclient = _connection_pool.get_client(self.conf)
try:
attr = getattr(superclient, attr_name)
try:
# Poke it to see if it's closed on the other end. This can happen if a connection
# sits in the connection pool longer than the read timeout of the server.
sock = self.conf.transport_mode != 'http' and _grab_transport_from_wrapper(superclient.transport).handle
if sock and create_synchronous_io_multiplexer().read([sock]):
# the socket is readable, meaning there is either data from a previous call
# (i.e our protocol is out of sync), or the connection was shut down on the
# remote side. Either way, we need to reopen the connection.
# If the socket was closed remotely, btw, socket.read() will return
# an empty string. This is a fairly normal condition, btw, since
# there are timeouts on both the server and client sides.
superclient.transport.close()
superclient.transport.open()
superclient.set_timeout(self.conf.timeout_seconds)
return attr(*args, **kwargs)
except TApplicationException, e:
# Unknown thrift exception... typically IO errors
logging.info("Thrift saw an application exception: " + str(e), exc_info=False)
raise StructuredException('THRIFTAPPLICATION', str(e), data=None, error_code=502)
except socket.error, e:
logging.info("Thrift saw a socket error: " + str(e), exc_info=False)
raise StructuredException('THRIFTSOCKET', str(e), data=None, error_code=502)
def create_job(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD", message=_('POST request required.'), error_code=405)
response = {
'status': 0,
'errors': None,
'job': None
}
if 'job' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving job'), data={'errors': 'job is missing.'}, error_code=400)
d = json.loads(smart_str(request.POST.get('job')))
job = client.Job.from_dict(d)
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE, ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['job'] = c.create_job(job).to_dict()
except RestException as e:
response.update(handle_rest_exception(e, _('Could not create job.')))
except SqoopException as e:
response['status'] = 100
response['errors'] = e.to_dict()
return JsonResponse(response)
def workflow_save(request, workflow):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
json_workflow = format_dict_field_values(
json.loads(request.POST.get('workflow')))
json_workflow.setdefault('schema_version', workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': form.errors},
error_code=400)
json_nodes = json_workflow['nodes']
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': errors},
error_code=400)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map,
request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node['child_links']
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def _validate_nodes_json(json_nodes, errors, user, workflow):
"""
Validates every node and link in the workflow.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
user is a User object that is associated with the node_type. Only needed for Subworkflow node.
workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
result = True
for node in json_nodes:
_errors = {}
node_dict = format_dict_field_values(node)
if node['node_type'] in ACTION_TYPES:
node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow)
else:
node_result = True
link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors)
result = result and node_result and link_result
if 'name' not in node and ( 'node_type' not in node or 'id' not in node ):
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400)
errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors
return result
def workflow(request, workflow):
if request.method != 'GET':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be GET request.'),
error_code=405)
return _workflow(request, workflow)
def submissions(request):
if request.method == 'GET':
return get_submissions(request)
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET request required.'),
error_code=405)
def create_link(request):
response = {'status': 0, 'errors': None, 'link': None}
if 'link' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving link'),
data={'errors': 'Link is missing.'},
error_code=400)
d = json.loads(smart_str(request.POST.get('link')))
link = client.Link.from_dict(d)
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['link'] = c.create_link(link).to_dict()
except RestException as e:
response.update(handle_rest_exception(e, _('Could not create link.')))
except SqoopException as e:
response['status'] = 100
response['errors'] = e.to_dict()
return JsonResponse(response)
def links(request):
if request.method == 'GET':
return get_links(request)
elif request.method == 'POST':
return create_link(request)
else:
raise StructuredException(code="INVALID_METHOD", message=_('GET or POST request required.'), error_code=405)
def workflows(request):
if request.method not in ['GET']:
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be GET request.'),
error_code=405)
if request.GET.get('managed', 'false').lower() == 'false':
extra = 'jobsub'
else:
extra = ''
workflow_docs = Document.objects.get_docs(request.user,
Workflow,
extra=extra)
response = {
'status': 0,
'data': {
'workflows': [
model_to_dict(workflow.content_object)
for workflow in workflow_docs
]
}
}
return HttpResponse(json.dumps(response), mimetype="application/json")
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving design'),
data={'errors': form.errors},
error_code=400)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = request.POST.get('name')
workflow.description = request.POST.get('description')
workflow.save()
doc = workflow.doc.get()
doc.extra = 'jobsub'
doc.save()
# Save design again to update all fields.
data = format_dict_field_values(request.POST.copy())
_save_design(workflow.id, data)
return get_design(request, workflow.id)
def restore_design(request, design_id):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
try:
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username,
_("Access denied: delete design %(id)s.") % {'id': design_id},
allow_root=True)
workflow.restore()
except Workflow.DoesNotExist:
LOG.error("Trying to restore non-existent workflow (id %s)" % (design_id,))
raise StructuredException(code="NOT_FOUND", message=_('Could not find design %s.') % design_id, error_code=404)
return render_json({
'status': 0
})
def connector(request, connector):
response = {'status': 0, 'errors': None, 'connector': None}
if request.method == 'GET':
response['connector'] = connector.to_dict()
return JsonResponse(response)
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET request required.'),
error_code=405)
def clone_design(request, design_id):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be a POST request.'), error_code=405)
workflow = _get_design(design_id)
clone = workflow.clone(request.fs, request.user)
cloned_action = clone.start.get_child('to')
cloned_action.name = clone.name
cloned_action.save()
return get_design(request, clone.id)
def update_job(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD", message=_('POST request required.'), error_code=405)
response = {
'status': 0,
'errors': None,
'job': None
}
if 'job' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving job'), data={'errors': 'job is missing.'}, error_code=400)
job.update_from_dict(json.loads(smart_str(request.POST.get('job'))))
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE)
response['job'] = c.update_job(job).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not update job.')))
def get_task(self, jobid, taskid):
"""Return a ThriftTaskInProgress"""
try:
tip = self.client.getTask(self.thread_local.request_context,
taskid)
except JobNotFoundException, e:
raise StructuredException(
code="JT_JOB_NOT_FOUND",
message="Could not find job %s on JobTracker." %
jobid.asString,
data=jobid)
def job(request, job):
response = {'status': 0, 'errors': None, 'job': None}
if request.method == 'GET':
response['job'] = job.to_dict()
return HttpResponse(json.dumps(response), mimetype="application/json")
elif request.method == 'POST':
return update_job(request, job)
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET or POST request required.'),
error_code=405)
def link(request, link):
response = {'status': 0, 'errors': None, 'link': None}
if request.method == 'GET':
response['link'] = link.to_dict()
return JsonResponse(response)
elif request.method == 'POST':
return update_link(request, link)
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET or POST request required.'),
error_code=405)
def workflow_actions(request, workflow):
if request.method != 'GET':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be GET request.'), error_code=405)
action_list = [action.get_full_node() for action in workflow.actions]
response = {
'status': 0,
'data': {
'actions': [model_to_dict(action) for action in action_list]
}
}
return JsonResponse(response)
def get_job(self, jobid):
"""
Returns a ThriftJobInProgress (including task info)
"""
try:
job = self.client.getJob(self.thread_local.request_context, jobid)
except JobNotFoundException, e:
raise StructuredException(
code="JT_JOB_NOT_FOUND",
message="Could not find job %s on JobTracker." %
jobid.asString,
data=jobid)
def _update_workflow_nodes_json(workflow, json_nodes, id_map, user):
"""Ideally would get objects from form validation instead."""
nodes = []
for json_node in json_nodes:
node = get_or_create_node(workflow, json_node, save=False)
if node.node_type == 'subworkflow':
try:
node.sub_workflow = Workflow.objects.get(
id=int(json_node['sub_workflow']))
except TypeError:
# sub_workflow is None
node.sub_workflow = None
except Workflow.DoesNotExist:
raise StructuredException(
code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': 'Chosen subworkflow does not exist.'},
error_code=400)
elif node.node_type == 'fork' and json_node['node_type'] == 'decision':
node.save(
) # Need to save in case database throws error when performing delete.
node = node.convert_to_decision()
node.save()
id_map[str(json_node['id'])] = node.id
for key in json_node:
if key == 'data':
if isinstance(json_node[key], basestring):
node.data = json_node[key]
else:
node.data = json.dumps(json_node[key])
elif key not in ('node_ptr', 'child_nodes', 'workflow', 'id',
'sub_workflow'):
setattr(node, key, format_field_value(key, json_node[key]))
node.workflow = workflow
node.save()
# Keep track of nodes in order of received list
# so that we may iterate over them again in the same order
# when we handle links
nodes.append(node)
# Delete unused nodes from workflow
old_nodes = Node.objects.filter(workflow=workflow).exclude(
id__in=map(lambda x: x.id, nodes))
for node in old_nodes:
node.get_full_node().delete()
return nodes
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id);
def job_stop(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'submission': None}
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username,
request.LANGUAGE_CODE)
response['submission'] = c.stop_job(job).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not stop job.')))
def link_delete(request, link):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD", message=_('POST request required.'), error_code=405)
response = {
'status': 0,
'errors': None
}
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE)
c.delete_link(link)
except RestException, e:
response.update(handle_rest_exception(e, _('Could not delete link.')))
def delete_context(request):
if request.method != 'DELETE':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a DELETE'))
response = {}
name = request.POST.get('name', '')
api = get_api(request.user)
try:
response = api.delete_context(name)
except ValueError:
# No json is returned
response = {'status': 'OK'}
except Exception, e:
response = json.loads(e.message)
def job_clone(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'job': None}
job.id = -1
job.name = '%s-copy' % job.name
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username,
request.LANGUAGE_CODE)
response['job'] = c.create_job(job).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not clone job.')))
def job_delete(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'job': None}
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
c.delete_job(job)
except RestException, e:
response.update(handle_rest_exception(e, _('Could not delete job.')))
def update_link(request, link):
response = {
'status': 0,
'errors': None,
'link': None
}
if 'link' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving link'), data={'errors': 'Link is missing.'}, error_code=400)
link.update_from_dict(json.loads(smart_str(request.POST['link'])))
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE)
response['link'] = c.update_link(link).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not update link.')))
def workflows(request):
if request.method not in ['GET']:
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be GET request.'),
error_code=405)
workflows = request.GET.get('managed', 'false').lower(
) == 'true' and Workflow.objects.managed() or Workflow.objects.unmanaged()
workflows_accessible = filter(
lambda x: Job.objects.can_read(request.user, x.id), workflows)
response = {
'status': 0,
'data': {
'workflows':
[model_to_dict(workflow) for workflow in workflows_accessible]
}
}
return HttpResponse(json.dumps(response), mimetype="application/json")
def __init__(self, msg, orig_exc=None):
# TODO(todd) use orig_exc for something fun
StructuredException.__init__(self,
"PERMISSION_DENIED",
msg)
