def test_parsing(self): assert_equal(['input', 'LIMIT', 'out'], find_dollar_variables(""" data = '$input'; $out = LIMIT data $LIMIT; -- ${nah} $output = STORE "$out"; """)) assert_equal(['max_salary', 'limit'], find_dollar_variables(""" SELECT sample_07.description, sample_07.salary FROM sample_07 WHERE ( sample_07.salary > $max_salary) ORDER BY sample_07.salary DESC LIMIT $limit"""))
def action_parameters(request): response = {'status': -1} parameters = set() try: node_data = json.loads(request.POST.get('node', '{}')) parameters = parameters.union(set(Node(node_data).find_parameters())) script_path = node_data.get('properties', {}).get('script_path', {}) if script_path: script_path = script_path.replace('hdfs://', '') if request.fs.do_as_user(request.user, request.fs.exists, script_path): data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024**2) if node_data['type'] in ('hive', 'hive2'): parameters = parameters.union( set(find_dollar_braced_variables(data))) elif node_data['type'] == 'pig': parameters = parameters.union( set(find_dollar_variables(data))) response['status'] = 0 response['parameters'] = list(parameters) except Exception, e: response['message'] = str(e)
def action_parameters(request): response = {'status': -1} parameters = set() try: node_data = json.loads(request.POST.get('node', '{}')) parameters = parameters.union(set(Node(node_data).find_parameters())) script_path = node_data.get('properties', {}).get('script_path', {}) if script_path: script_path = script_path.replace('hdfs://', '') if request.fs.do_as_user(request.user, request.fs.exists, script_path): data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2) if node_data['type'] in ('hive', 'hive2'): parameters = parameters.union(set(find_dollar_braced_variables(data))) elif node_data['type'] == 'pig': parameters = parameters.union(set(find_dollar_variables(data))) elif node_data['type'] == 'hive-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str()))) elif node_data['type'] == 'sqoop-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str()))) elif node_data['type'] == 'spark-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']: parameters = parameters.union(set(find_dollar_braced_variables(arg))) response['status'] = 0 response['parameters'] = list(parameters) except Exception, e: response['message'] = str(e)
def action_parameters(request): response = {'status': -1} parameters = set() try: node_data = json.loads(request.POST.get('node', '{}')) parameters = parameters.union(set(Node(node_data).find_parameters())) script_path = node_data.get('properties', {}).get('script_path', {}) if script_path: script_path = script_path.replace('hdfs://', '') if request.fs.do_as_user(request.user, request.fs.exists, script_path): data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2) if node_data['type'] in ('hive', 'hive2'): parameters = parameters.union(set(find_dollar_braced_variables(data))) elif node_data['type'] == 'pig': parameters = parameters.union(set(find_dollar_variables(data))) response['status'] = 0 response['parameters'] = list(parameters) except Exception, e: response['message'] = str(e)