def clone_design(request, design_id):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be a POST request.'),
error_code=405)
workflow = _get_design(request.user, design_id)
clone = workflow.clone(request.fs, request.user)
doc = clone.doc.get()
doc.extra = 'jobsub'
doc.save()
cloned_action = clone.start.get_child('to')
cloned_action.name = clone.name
cloned_action.save()
return get_design(request, clone.id)
def job_clone(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'job': None}
job.id = -1
job.name = '%s-copy' % job.name
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username,
request.LANGUAGE_CODE)
response['job'] = c.create_job(job).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not clone job.')))
def update_link(request, link):
response = {
'status': 0,
'errors': None,
'link': None
}
if 'link' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving link'), data={'errors': 'Link is missing.'}, error_code=400)
link.update_from_dict(json.loads(smart_str(request.POST['link'])))
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE)
response['link'] = c.update_link(link).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not update link.')))
def link_delete(request, link):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None}
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
c.delete_link(link)
except RestException, e:
response.update(handle_rest_exception(e, _('Could not delete link.')))
def job_stop(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'submission': None}
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['submission'] = c.stop_job(job).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not stop job.')))
def _update_workflow_nodes_json(workflow, json_nodes, id_map, user):
"""Ideally would get objects from form validation instead."""
nodes = []
for json_node in json_nodes:
node = get_or_create_node(workflow, json_node, save=False)
if node.node_type == 'subworkflow':
try:
node.sub_workflow = Workflow.objects.get(
id=int(json_node['sub_workflow']))
except Workflow.DoesNotExist:
raise StructuredException(
code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': 'Chosen subworkflow does not exist.'},
error_code=400)
elif node.node_type == 'fork' and json_node['node_type'] == 'decision':
node.save(
) # Need to save in case database throws error when performing delete.
node = node.convert_to_decision()
node.save()
id_map[str(json_node['id'])] = node.id
for key in json_node:
if key not in ('node_ptr', 'child_nodes', 'workflow', 'id',
'sub_workflow'):
setattr(node, key, format_field_value(key, json_node[key]))
node.workflow = workflow
node.save()
# Keep track of nodes in order of received list
# so that we may iterate over them again in the same order
# when we handle links
nodes.append(node)
# Delete unused nodes from workflow
old_nodes = Node.objects.filter(workflow=workflow).exclude(
id__in=map(lambda x: x.id, nodes))
for node in old_nodes:
node.get_full_node().delete()
return nodes
def update_connection(request, connection):
response = {'status': 0, 'errors': None, 'connection': None}
if 'connection' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving connection'),
data={'errors': 'Connection is missing.'},
error_code=400)
connection.update_from_dict(json.loads(request.POST['connection']))
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username,
request.LANGUAGE_CODE)
response['connection'] = c.update_connection(connection).to_dict()
except RestException, e:
response.update(
handle_rest_exception(e, _('Could not update connection.')))
def workflows(request):
if request.method not in ['GET']:
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be GET request.'),
error_code=405)
workflows = request.GET.get('managed', 'false').lower(
) == 'true' and Workflow.objects.managed() or Workflow.objects.unmanaged()
workflows_accessible = filter(
lambda x: Job.objects.can_read(request.user, x.id), workflows)
response = {
'status': 0,
'data': {
'workflows':
[model_to_dict(workflow) for workflow in workflows_accessible]
}
}
return HttpResponse(json.dumps(response), mimetype="application/json")
def driver(request):
response = {'status': 0, 'errors': None, 'driver': None}
if request.method == 'GET':
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['driver'] = c.get_driver().to_dict()
except RestException as e:
response.update(
handle_rest_exception(e, _('Could not get driver.')))
return JsonResponse(response)
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET request required.'),
error_code=405)
def job_delete(request, job):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD", message=_('POST request required.'), error_code=405)
response = {
'status': 0,
'errors': None,
'job': None
}
try:
c = client.SqoopClient(conf.SERVER_URL.get(), request.user.username, request.LANGUAGE_CODE, ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
c.delete_job(job)
except RestException as e:
response.update(handle_rest_exception(e, _('Could not delete job.')))
except SqoopException as e:
response['status'] = 100
response['errors'] = e.to_dict()
return JsonResponse(response)
def workflows(request):
if request.method not in ['GET']:
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be GET request.'), error_code=405)
if request.GET.get('managed', 'false').lower() == 'false':
extra='jobsub'
else:
extra=''
workflow_docs = Document.objects.get_docs(request.user, Workflow, extra=extra)
response = {
'status': 0,
'data': {
'workflows': [model_to_dict(workflow.content_object) for workflow in workflow_docs]
}
}
return JsonResponse(response)
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(
request, workflow.owner.username,
_("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None),
request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving design'),
data={'errors': form.errors},
error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id)
def create_context(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a POST'))
response = {}
name = request.POST.get('name', '')
memPerNode = request.POST.get('mem-per-node', '512m')
numCores = request.POST.get('num-cpu-cores', '1')
api = get_api(request.user)
try:
response = api.create_context(name,
memPerNode=memPerNode,
numCores=numCores)
except ValueError:
# No json is returned
response = {'status': 'OK'}
except Exception, e:
response = json.loads(e.message)
def create_link(request):
response = {'status': 0, 'errors': None, 'link': None}
if 'link' not in request.POST:
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving link'),
data={'errors': 'Link is missing.'},
error_code=400)
d = json.loads(smart_str(request.POST.get('link')))
link = client.Link.from_dict(d)
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['link'] = c.create_link(link).to_dict()
except RestException, e:
response.update(handle_rest_exception(e, _('Could not create link.')))
def get_or_create_node(workflow, node_data, save=True):
node = None
id = str(node_data['id'])
separator_index = id.find(':')
if separator_index == -1:
return Node.objects.get(id=id, workflow=workflow).get_full_node()
node_type = id[0:separator_index]
node_model = NODE_TYPES.get(node_type, None)
kwargs = {'workflow': workflow, 'node_type': node_data['node_type']}
if node_model:
node = node_model(**kwargs)
else:
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Could not find node of type'), data=node_data, error_code=500)
if save:
node.save()
return node
def _wrap_callable(self, attr_name):
# It's gonna be a thrift call. Add wrapping logic to reopen the transport,
# and return the connection to the pool when done.
def wrapper(*args, **kwargs):
superclient = _connection_pool.get_client(self.conf)
try:
attr = getattr(superclient, attr_name)
try:
# Poke it to see if it's closed on the other end. This can happen if a connection
# sits in the connection pool longer than the read timeout of the server.
sock = self.conf.transport_mode != 'http' and _grab_transport_from_wrapper(superclient.transport).handle
if sock and create_synchronous_io_multiplexer().read([sock]):
# the socket is readable, meaning there is either data from a previous call
# (i.e our protocol is out of sync), or the connection was shut down on the
# remote side. Either way, we need to reopen the connection.
# If the socket was closed remotely, btw, socket.read() will return
# an empty string. This is a fairly normal condition, btw, since
# there are timeouts on both the server and client sides.
superclient.transport.close()
superclient.transport.open()
superclient.set_timeout(self.conf.timeout_seconds)
logging.debug("Thrift client %s call" % superclient)
return attr(*args, **kwargs)
except TApplicationException, e:
# Unknown thrift exception... typically IO errors
logging.info("Thrift saw an application exception: " + str(e), exc_info=False)
raise StructuredException('THRIFTAPPLICATION', str(e), data=None, error_code=502)
except socket.error, e:
logging.info("Thrift saw a socket error: " + str(e), exc_info=False)
raise StructuredException('THRIFTSOCKET', str(e), data=None, error_code=502)
except TTransportException, e:
err_msg = str(e)
logging.info("Thrift saw a transport exception: " + err_msg, exc_info=False)
if err_msg and 'generic failure: Unable to find a callback: 32775' in err_msg:
raise StructuredException(_("Increase the sasl_max_buffer value in hue.ini"), err_msg, data=None, error_code=502)
raise StructuredThriftTransportException(e, error_code=502)
def upload_app(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a POST'))
response = {'status': -1}
form = UploadApp(request.POST, request.FILES)
if form.is_valid():
app_name = form.cleaned_data['app_name']
try:
data = form.cleaned_data['jar_file'].read()
api = get_api(request.user)
response['status'] = 0
response['results'] = api.upload_jar(app_name, data)
except ValueError:
# No json is returned
pass
else:
response['results'] = form.errors
return redirect(request.META['HTTP_REFERER'])
def link_clone(request, link):
if request.method != 'POST':
raise StructuredException(code="INVALID_METHOD",
message=_('POST request required.'),
error_code=405)
response = {'status': 0, 'errors': None, 'link': None}
link.id = -1
link.name = '%s-copy' % link.name
try:
c = client.SqoopClient(
conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE,
ssl_cert_ca_verify=conf.SSL_CERT_CA_VERIFY.get())
response['link'] = c.create_link(link).to_dict()
except RestException as e:
response.update(handle_rest_exception(e, _('Could not clone link.')))
except SqoopException as e:
response['status'] = 100
response['errors'] = e.to_dict()
return JsonResponse(response)
def _validate_nodes_json(json_nodes, errors, user, workflow):
"""
Validates every node and link in the workflow.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
user is a User object that is associated with the node_type. Only needed for Subworkflow node.
workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
result = True
for node in json_nodes:
_errors = {}
node_dict = format_dict_field_values(node)
if node['node_type'] in ACTION_TYPES:
node_result = _validate_node_json(node['node_type'], node_dict,
_errors, user, workflow)
else:
node_result = True
link_result = _validate_node_links_json(node['node_type'],
node_dict['child_links'],
_errors)
result = result and node_result and link_result
if not node.has_key('name') and (not node.has_key('node_type')
or not node.has_key('id')):
raise StructuredException(
code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': 'Node is missing a name.'},
error_code=400)
errors[node.get('name', '%s-%s' %
(node.get('node_type'), node.get('id')))] = _errors
return result
for index in range(0, len(node_list)):
nodes[index]['child_links'] = [
model_to_dict(link)
for link in node_list[index].get_all_children_links()
]
workflow_dict['nodes'] = nodes
response['status'] = 0
response['data'] = workflow_dict
return HttpResponse(json.dumps(response), mimetype="application/json")
@error_handler
@check_job_access_permission(exception_class=(lambda x: StructuredException(
code="UNAUTHORIZED_REQUEST_ERROR", message=x, data=None, error_code=401)))
@check_job_edition_permission(exception_class=(lambda x: StructuredException(
code="UNAUTHORIZED_REQUEST_ERROR", message=x, data=None, error_code=401)))
def workflow_validate_node(request, workflow, node_type):
response = {'status': -1, 'data': {}}
node_dict = format_dict_field_values(json.loads(request.POST.get('node')))
if _validate_node_json(node_type, node_dict, response['data'],
request.user, workflow):
response['status'] = 0
else:
response['status'] = -1
return HttpResponse(json.dumps(response), mimetype="application/json")
def submissions(request):
if request.method == 'GET':
return get_submissions(request)
else:
raise StructuredException(code="INVALID_METHOD", message=_('GET request required.'), error_code=405)
from django.utils.translation import ugettext as _
from sqoop import client, conf
from desktop.lib.exceptions import StructuredException
from desktop.lib.rest.http_client import RestException
from exception import handle_rest_exception
from django.views.decorators.cache import never_cache
__all__ = ['framework']
LOG = logging.getLogger(__name__)
@never_cache
def framework(request):
response = {'status': 0, 'errors': None, 'framework': None}
if request.method == 'GET':
try:
c = client.SqoopClient(conf.SERVER_URL.get(),
request.user.username,
request.LANGUAGE_CODE)
response['framework'] = c.get_framework().to_dict()
except RestException, e:
response.update(
handle_rest_exception(e, _('Could not get framework.')))
return HttpResponse(json.dumps(response), mimetype="application/json")
else:
raise StructuredException(code="INVALID_METHOD",
message=_('GET request required.'),
error_code=405)
def wrapper(*args, **kwargs):
tries_left = 3
while tries_left:
# clear exception state so our re-raise can't reraise something
# old. This isn't strictly necessary, but feels safer.
# py3 doesn't have this
if sys.version_info[0] == 2:
sys.exc_clear()
try:
if not self.transport.isOpen():
self.transport.open()
st = time.time()
str_args = _unpack_guid_secret_in_handle(repr(args))
logging.debug("Thrift call: %s.%s(args=%s, kwargs=%s)" %
(str(self.wrapped.__class__), attr, str_args,
repr(kwargs)))
ret = res(*args, **kwargs)
if ENABLE_SMART_THRIFT_POOL.get(
) and 'OpenSession' == attr and 'http_addr' in repr(ret):
coordinator_host = re.search(
'http_addr\':\ \'(.*:[0-9]{2,})\', \'', repr(ret))
self.coordinator_host = coordinator_host.group(1)
log_msg = _unpack_guid_secret_in_handle(repr(ret))
# Truncate log message, increase output in DEBUG mode
log_limit = 2000 if settings.DEBUG else 1000
log_msg = log_msg[:log_limit] + (log_msg[log_limit:]
and '...')
duration = time.time() - st
# Log the duration at different levels, depending on how long it took.
logmsg = "Thrift call: %s.%s(args=%s, kwargs=%s) returned in %dms: %s" % (
str(self.wrapped.__class__), attr, str_args,
repr(kwargs), duration * 1000, log_msg)
log_if_slow_call(duration=duration, message=logmsg)
return ret
except (socket.error, socket.timeout,
TTransportException) as e:
self.transport.close()
if isinstance(e, socket.timeout
) or 'read operation timed out' in str(
e): # Can come from ssl.SSLError
logging.warn(
"Not retrying thrift call %s due to socket timeout"
% attr)
raise
else:
tries_left -= 1
if tries_left:
logging.info("Thrift exception; retrying: " +
str(e),
exc_info=0)
if 'generic failure: Unable to find a callback: 32775' in str(
e):
logging.warn(
"Increase the sasl_max_buffer value in hue.ini"
)
else:
raise
except Exception as e:
logging.exception(
"Thrift saw exception (this may be expected).")
if "'client_protocol' is unset" in e.message:
raise StructuredException(
'OPEN_SESSION',
'Thrift version configured by property thrift_version might be too high. Request failed with "%s"'
% str(e),
data=None,
error_code=502)
else:
raise
logging.warn("Out of retries for thrift call: " + attr)
raise
def process_exception(self, request, exception):
if isinstance(exception, SubmissionError) and not hasattr(
SubmissionError, "response_data"):
raise StructuredException(code="SUBMISSION_ERROR",
message=exception.message)
nodes = [model_to_dict(node) for node in node_list]
for index in range(0, len(node_list)):
nodes[index]['child_links'] = [
model_to_dict(link)
for link in node_list[index].get_all_children_links()
]
workflow_dict['nodes'] = nodes
response['status'] = 0
response['data'] = workflow_dict
return HttpResponse(json.dumps(response), mimetype="application/json")
@check_job_access_permission(exception_class=(lambda x: StructuredException(
code="UNAUTHORIZED_REQUEST_ERROR", message=x, data=None, error_code=401)))
@check_job_edition_permission(exception_class=(lambda x: StructuredException(
code="UNAUTHORIZED_REQUEST_ERROR", message=x, data=None, error_code=401)))
def workflow_validate_node(request, workflow, node_type):
response = {'status': -1, 'data': {}}
node_dict = format_dict_field_values(
json.loads(str(request.POST.get('node'))))
if _validate_node_json(node_type, node_dict, response['data'],
request.user, workflow):
response['status'] = 0
else:
response['status'] = -1
return HttpResponse(json.dumps(response), mimetype="application/json")
def get_task(self, jobid, taskid):
"""Return a ThriftTaskInProgress"""
try:
tip = self.client.getTask(self.thread_local.request_context,
taskid)
except JobNotFoundException, e:
raise StructuredException(
code="JT_JOB_NOT_FOUND",
message="Could not find job %s on JobTracker." %
jobid.asString,
data=jobid)
except TaskNotFoundException, e:
raise StructuredException(
code="JT_TASK_NOT_FOUND",
message="Could not find task %s on JobTracker." %
taskid.asString,
data=taskid)
self._fixup_task_in_progress(tip)
return tip
def get_current_time(self):
"""
Returns an integer timestamp
"""
return self.client.getCurrentTime(self.thread_local.request_context)
def get_job_xml(self, jobid):
"""
Returns a string representation of the job XML
"""
def workflow(request, workflow):
if request.method != 'GET':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be GET request.'), error_code=405)
return _workflow(request, workflow)
