def test_parsing(self):
assert_equal(['input', 'LIMIT', 'out'], find_dollar_variables("""
data = '$input';
$out = LIMIT data $LIMIT; -- ${nah}
$output = STORE "$out";
"""))
assert_equal(['max_salary', 'limit'], find_dollar_variables("""
SELECT sample_07.description, sample_07.salary
FROM
sample_07
WHERE
( sample_07.salary > $max_salary)
ORDER BY sample_07.salary DESC
LIMIT $limit"""))
def test_parsing(self):
assert_equal(['input', 'LIMIT', 'out'], find_dollar_variables("""
data = '$input';
$out = LIMIT data $LIMIT; -- ${nah}
$output = STORE "$out";
"""))
assert_equal(['max_salary', 'limit'], find_dollar_variables("""
SELECT sample_07.description, sample_07.salary
FROM
sample_07
WHERE
( sample_07.salary > $max_salary)
ORDER BY sample_07.salary DESC
LIMIT $limit"""))
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists,
script_path):
data = request.fs.do_as_user(request.user, request.fs.read,
script_path, 0, 16 * 1024**2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(
set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(
set(find_dollar_variables(data)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists, script_path):
data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(set(find_dollar_variables(data)))
elif node_data['type'] == 'hive-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'sqoop-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'spark-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']:
parameters = parameters.union(set(find_dollar_braced_variables(arg)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists, script_path):
data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(set(find_dollar_variables(data)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists, script_path):
data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(set(find_dollar_variables(data)))
elif node_data['type'] == 'hive-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'sqoop-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'spark-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']:
parameters = parameters.union(set(find_dollar_braced_variables(arg)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
