def test_spray_file_string(self):
thor_file = '~thor::test_spray_file_string'
overwrite = True
expire = 1
delete_workunit = True
chunk_size = 10000
max_workers = 3
conn = hpycc.Connection("user", test_conn=False)
col_1_values = ['1', '3', '5', '6']
col_2_values = ['aa', 'ab', 'ac', 'ad']
df = pd.DataFrame({
"a": col_1_values,
"b": col_2_values
}).sort_values('a')
with TemporaryDirectory() as d:
p = os.path.join(d, "test.csv")
df.to_csv(p, index=False)
spray_file(conn, p, thor_file, overwrite, expire, chunk_size,
max_workers, delete_workunit)
res = get_thor_file(connection=conn, thor_file=thor_file)[['a', 'b']]
pd.testing.assert_frame_equal(df, res)
def test_get_logical_file_chunk_fails_with_no_json(self, mock):
mock.return_value = requests.Response()
conn = hpycc.Connection("user", server="aa", port=123, test_conn=False)
with TemporaryDirectory() as td:
p = os.path.join(td, "data.csv")
with self.assertRaises(simpleJSONDecodeError):
conn.get_logical_file_chunk("file", 1, 2, 1, 0)
def test_check_syntax_passes_with_good_script(self):
conn = hpycc.Connection("user", test_conn=False)
with TemporaryDirectory() as d:
p = os.path.join(d, "tmp.ecl")
with open(p, "w+") as file:
file.write("output(2);")
self.assertIsNone(conn.check_syntax(p))
def test_run_url_request_returns_response_after_single_error(self, mock):
r = requests.Response()
r.status_code = 200
mock.side_effect = (ValueError, r)
conn = hpycc.Connection("user", test_conn=False)
result = conn.run_url_request("dfsd.dfd", max_attempts=3, max_sleep=0)
self.assertIsInstance(result, requests.Response)
def setUpClass(cls):
cls.conn = hpycc.Connection("user")
script = ("OUTPUT(2);OUTPUT('a');OUTPUT(DATASET([{1, 'a'}], "
"{INTEGER a; STRING b;}), NAMED('ds'));")
cls.res = _get_outputs_from_ecl_string(cls.conn, script)
a = [{
"col": "alltrue",
"content": "[{true}, {true}, {true}]",
"coltype": "BOOLEAN"
}, {
"col": "allfalse",
"content": "[{false}, {false}, {false}]",
"coltype": "BOOLEAN"
}, {
"col": "trueandfalse",
"content": "[{true}, {false}, {true}]",
"coltype": "BOOLEAN"
}, {
"col": "truefalsestrings",
"content": "[{'true'}, {'false'}, {'false'}]",
"coltype": "STRING"
}, {
"col": "truefalseblank",
"content": "[{'true'}, {'false'}, {''}]",
"coltype": "STRING"
}]
f = [("OUTPUT(DATASET({0[content]}, {{{0[coltype]} {0[col]};}}), "
"NAMED('{0[col]}'));").format(i) for i in a]
cls.t_f_res = _get_outputs_from_ecl_string(cls.conn, "\n".join(f))
def test_concatenate_logical_files_concatenates_one_file(self):
thor_file = '~thor::test_concatenate_logical_files_concatenates_one_file'
overwrite = True
expire = 1
delete_workunit = True
conn = hpycc.Connection("user", test_conn=False)
output_names = ['~a2']
col_1_values = ['1']
col_2_values = ['aa']
[
send_file_chunks(conn, CONCAT_SCRIPT_BASE % (col1, col2, nam)) for
col1, col2, nam in zip(col_1_values, col_2_values, output_names)
]
_concatenate_logical_files(conn, output_names, thor_file,
'STRING a; STRING b;', overwrite, expire,
delete_workunit)
res = get_thor_file(connection=conn,
thor_file=thor_file)[['a', 'b']].sort_values("a")
expected_result = pd.DataFrame({
"a": col_1_values,
"b": col_2_values
}).sort_values("a")
pd.testing.assert_frame_equal(expected_result, res)
def test_get_logical_file_chunk_uses_correct_url(self, mock):
mock.json.return_value = {"WUResultResponse": 123}
conn = hpycc.Connection("user", server="aa", port=123, test_conn=False)
with TemporaryDirectory() as td:
p = os.path.join(td, "data.csv")
conn.get_logical_file_chunk("file", 1, 2, 1, 0)
mock.assert_called()
def test_run_url_request_counts_500_as_error(self, mock):
bad_response = requests.Response()
bad_response.status_code = 500
mock.return_value = bad_response
conn = hpycc.Connection("user", test_conn=False)
with self.assertRaises(requests.exceptions.RetryError):
conn.run_url_request("dfsd.dfd", max_attempts=2, max_sleep=0)
def test_run_ecl_script_command_uses_repo_if_list(self, mock):
conn = hpycc.Connection("user", test_conn=False, repo=["C:", "D:"])
conn.run_ecl_script("test.ecl", syntax_check=False,
delete_workunit=False, stored={})
mock.assert_called_with(['ecl', 'run', '-v', '--server=localhost',
'--port=8010', '--username=user',
'--password=password',
'thor', 'test.ecl', "-I=C:", "-I=D:"])
def test_run_ecl_script_command_uses_legacy_if_none(self, mock):
conn = hpycc.Connection("user", legacy=None, test_conn=False)
conn.run_ecl_script("test.ecl", syntax_check=False,
delete_workunit=False, stored={})
mock.assert_called_with(['ecl', 'run', '-v', '--server=localhost',
'--port=8010', '--username=user',
'--password=password',
'thor', 'test.ecl'])
def test_check_syntax_fails_with_errors(self):
conn = hpycc.Connection("user", test_conn=False)
with TemporaryDirectory() as d:
p = os.path.join(d, "tmp.ecl")
with open(p, "w+") as file:
file.write("sdfdsf")
with self.assertRaises(SyntaxError):
conn.check_syntax(p)
def test_run_url_request_uses_all_attempts(self, mock):
final_response = requests.Response()
final_response.status_code = 200
side_effects = (ValueError, ValueError, ValueError, ValueError, final_response)
mock.side_effect = side_effects
conn = hpycc.Connection("user", test_conn=False)
conn.run_url_request("dfsd.dfd", max_attempts=5, max_sleep=3)
self.assertEqual(mock.call_count, 5)
def test_run_script_passes_syntax_check_with_good_script(self, mock):
conn = hpycc.Connection("user", test_conn=False, repo=None)
good_script = "output(2);"
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
conn.run_ecl_script(p, syntax_check=True, delete_workunit=False, stored={})
mock.assert_called()
def test_run_ecl_script_command_uses_server_port_and_username(self, mock):
conn = hpycc.Connection("user", test_conn=False, server="abc",
port=123)
conn.run_ecl_script("test.ecl", syntax_check=False,
delete_workunit=False, stored={})
mock.assert_called_with(['ecl', 'run', '-v', '--server=abc',
'--port=123', '--username=user',
'--password=password',
'thor', 'test.ecl'])
def test_check_syntax_returns_syntax_errors(self):
conn = hpycc.Connection("user", test_conn=False)
expected_err = "error C3002"
with TemporaryDirectory() as d:
p = os.path.join(d, "tmp.ecl")
with open(p, "w+") as file:
file.write("sdfdsf")
with self.assertRaisesRegex(SyntaxError, expected_err):
conn.check_syntax(p)
def test_run_script_returns_true(self):
conn = hpycc.Connection("user", test_conn=False)
good_script = "OUTPUT(2);"
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
res = hpycc.run_script(conn, p, delete_workunit=False)
self.assertTrue(res)
def test_run_ecl_script_command_works_with_special_password_chars(
self, mock):
conn = hpycc.Connection("user", password="******", test_conn=False)
conn.run_ecl_script("test.ecl", syntax_check=False,
delete_workunit=False, stored={})
mock.assert_called_with(['ecl', 'run', '-v', '--server=localhost',
'--port=8010', '--username=user',
'--password=a\nb',
'thor', 'test.ecl'])
def test_run_script_fails_if_server_unavailable(self):
conn = hpycc.Connection("user", test_conn=False, repo=None)
good_script = "output(2);"
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
with self.assertRaises(subprocess.SubprocessError):
conn.run_ecl_script(p, syntax_check=False,
delete_workunit=False, stored={})
def test_run_script_fails_syntax_check_with_bad_script(self):
conn = hpycc.Connection("user", test_conn=False, repo=None)
bad_script = "sdf"
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(bad_script)
with self.assertRaises(SyntaxError):
conn.run_ecl_script(p, syntax_check=True,
delete_workunit=False, stored={})
def test_run_script_does_not_delete_workunit(self):
conn = hpycc.Connection("user", test_conn=False)
check_wu_ecl = "ecl getwuid -u user -pw 1234 -s localhost -n test_run_script_deletes_workunit"
good_script = ("#WORKUNIT('name','test_run_script_deletes_workunit');"
"OUTPUT(2);")
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
hpycc.run_script(conn, p, delete_workunit=False)
res = conn._run_command(check_wu_ecl)
self.assertRegex(res.stdout, "W[0-9]{8}-[0-9]{6}")
def test_run_script_does_nothing_when_not_using_stored(self):
conn = hpycc.Connection("user", test_conn=False)
file_name = "test_run_script_does_nothing_when_not_using_stored_"
good_script = "str := 'abc' : STORED('str');" \
" z := DATASET([{{str + str}}]," \
" {{STRING str;}}); OUTPUT(z,,'~{}', " \
"EXPIRE(1));".format(file_name)
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
hpycc.run_script(conn, p, stored={'b': 'Shouldbethecorrectoutput'})
res = hpycc.get_thor_file(conn, file_name)
expected = pd.DataFrame({"str": ["abcabc"], "__fileposition__": [0]})
pd.testing.assert_frame_equal(expected.sort_index(axis=1), res.sort_index(axis=1), check_dtype=False)
def test_run_script_saves_logical_file(self):
conn = hpycc.Connection("user", test_conn=False)
good_script = "\n".join([
"a:= DATASET([{'1', 'a'}], "
"{STRING a; STRING b});",
"output(a, ,'testrunscriptsaveslogicalfile');"
])
with TemporaryDirectory() as d:
p = os.path.join(d, "test.ecl")
with open(p, "w+") as file:
file.write(good_script)
res = conn.run_ecl_script(p, syntax_check=True,
delete_workunit=False, stored={})
self.assertTrue(res)
res = conn.get_logical_file_chunk("thor::testrunscriptsaveslogicalfile", 0, 1, 3, 1)
expected = {"a": ["1"], "b": ["a"], "__fileposition__": ["0"]}
self.assertEqual(expected, res)
def test_check_syntax_passes_with_warnings(self):
conn = hpycc.Connection("user", test_conn=False)
warning_string = "\n".join([
"a := DATASET([{1,2}], {INTEGER a; INTEGER b});",
"a;",
"b := DATASET([{1,2}], {INTEGER a; INTEGER b});",
"b;",
"groupREC := RECORD",
"STRING a := a.a;",
"STRING b := a.b;",
"INTEGER n := count(group);",
"END;",
"TABLE(a, groupREC, a)"
])
with TemporaryDirectory() as d:
p = os.path.join(d, "tmp.ecl")
with open(p, "w+") as file:
file.write(warning_string)
self.assertIsNone(conn.check_syntax(p))
def test_spray_file_df(self):
thor_file = '~thor::test_spray_file_df'
overwrite = True
expire = 1
delete_workunit = True
chunk_size = 10000
max_workers = 3
conn = hpycc.Connection("user", test_conn=False)
col_1_values = ['1', '3', '5', '6']
col_2_values = ['aa', 'ab', 'ac', 'ad']
df = pd.DataFrame({
"a": col_1_values,
"b": col_2_values
}).sort_values('a')
spray_file(conn, df, thor_file, overwrite, expire, chunk_size,
max_workers, delete_workunit)
res = get_thor_file(connection=conn, thor_file=thor_file)[['a', 'b']]
pd.testing.assert_frame_equal(df, res)
def test_spray_file_string_smallest_chunks_many_workers(self):
thor_file = '~test_spray_file_string_smallest_chunks_many_workers'
overwrite = True
expire = 1
delete_workunit = True
chunk_size = 1
max_workers = 100
conn = hpycc.Connection("user", test_conn=False)
col_1_values = ['1', '3', '5', '7', '9', '11', '13']
col_2_values = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
df = pd.DataFrame({
"a": col_1_values,
"b": col_2_values
}).sort_values('a').reset_index(drop=True)
spray_file(conn, df, thor_file, overwrite, expire, chunk_size,
max_workers, delete_workunit)
res = get_thor_file(connection=conn, thor_file=thor_file)[[
'a', 'b'
]].sort_values('a').reset_index(drop=True)
pd.testing.assert_frame_equal(df, res)
def setUp(self):
self.conn = hpycc.Connection("user")
def setUp(self):
self.conn = hpycc.Connection("user", test_conn=False)
def test_test_conn_false(self, mock):
hpycc.Connection("user", test_conn=False)
self.assertFalse(mock.called)
def test_test_conn_default(self, mock):
hpycc.Connection("user")
mock.assert_called()
def test_password_raises_error_if_spaces(self):
with self.assertRaises(AttributeError):
hpycc.Connection("a", password="******", test_conn=False)
