To run this Python script you need to install livy-python-api-*version*.tar.gz with
easy_install first.
python /pathTo/pi_app.py http://<livy-server>:8998 2
"""
if len(sys.argv) != 3:
print("Usage: pi_app <livy url> <slices>", file=sys.stderr)
exit(-1)
slices = int(sys.argv[2])
samples = 100000 * slices
client = HttpClient(sys.argv[1])
def f(_):
x = random() * 2 - 1
y = random() * 2 - 1
return 1 if x ** 2 + y ** 2 <= 1 else 0
def pi_job(context):
count = context.sc.parallelize(range(1, samples + 1), slices).map(f).reduce(add)
return 4.0 * count / samples
pi = client.submit(pi_job).result()
print("Pi is roughly %f" % pi)
client.stop(True)
class LivyHandler:
def __init__(self, host='http://localhost:8998'):
self._headers = {'Content-Type': 'application/json'}
if host is not None:
self.create_pyspark_session(host)
def _wait_for_state(self, url, desired_state):
while True:
r = requests.get(url, headers=self._headers)
if r.json()['state'] == desired_state:
break
time.sleep(1)
return r
def create_pyspark_session(self, host):
self._host = host
data = {'kind': 'pyspark'}
# Create a Spark session
print('Creating Spark session...')
r = requests.post(host + '/sessions',
data=json.dumps(data),
headers=self._headers)
# Wait until the new Spark session is ready to use
self._session_url = host + r.headers['location']
r = self._wait_for_state(self._session_url, 'idle')
# Create client for Livy batch jobs
self._lc = HttpClient(self._session_url)
def exec_str(self, code):
print('Submitting code...')
statements_url = self._session_url + '/statements'
data = {'code': code}
r = requests.post(statements_url,
data=json.dumps(data),
headers=self._headers)
# Wait until the code completes
print('Running code...')
status_url = self._host + r.headers['location']
r = self._wait_for_state(status_url, 'available')
output = r.json()['output']
print('output=', output)
if output['status'] == 'error':
ans = {'text/plain': output['traceback']}
else:
ans = {'text/plain': [output['data']['text/plain']]}
return ans
def exec_file(self, py_uri):
py_uri_abs = abspath(py_uri)
self._lc.upload_pyfile(py_uri_abs)
m = splitext(basename(py_uri_abs))[0]
try:
m_imp = import_module(m)
except ImportError:
raise
def upload_pyfile_job(jc):
return m_imp.main(jc.sc)
return self._lc.submit(upload_pyfile_job).result()
def close(self):
print('Closing Spark session...')
requests.delete(self._session_url, headers=self._headers)
To run this Python script you need to install livy-python-api-*version*.tar.gz with
easy_install first.
python /pathTo/pi_app.py http://<livy-server>:8998 2
"""
if len(sys.argv) != 3:
print("Usage: pi_app <livy url> <slices>", file=sys.stderr)
exit(-1)
slices = int(sys.argv[2])
samples = 100000 * slices
client = HttpClient(sys.argv[1])
def f(_):
x = random() * 2 - 1
y = random() * 2 - 1
return 1 if x**2 + y**2 <= 1 else 0
def pi_job(context):
count = context.sc.parallelize(range(1, samples + 1),
slices).map(f).reduce(add)
return 4.0 * count / samples
pi = client.submit(pi_job).result()
print("Pi is roughly %f" % pi)
client.stop(True)