def get_file(id):
"""
Handles any page in the template folder
:return:
"""
# show the user profile for that user
dml = DataManager.OutputDataManager(connection)
output = dml.get_file(id)
return JSONOutputConverter.getString(output)
def job_status(fileid, execid):
"""
get the status
:return:
"""
output = dict()
dml = DataManager.OutputDataManager(connection)
sessionid = None
# get the session id from execution details
exec_output = dml.get_exec_details(execid)
if exec_output["status"] == "success":
filename = exec_output["result"]["fileid"]
sessionid = exec_output["result"]["sessionid"]
arguments = exec_output["result"]["arguments"]
else:
output["status"] = "fail"
output["result"] = dict()
output["result"]["message"] = "Session id not found"
status_output = livyquery.job_status(sessionid)
if status_output.status_code == 404:
output["status"] = "fail"
return JSONOutputConverter.getString(output)
# get log status using livy
livy_log = livyquery.log_status(sessionid)
if livy_log.status_code == 404:
output["status"] = "fail"
output["result"] = dict()
output["result"]["message"] = "Session not found"
return JSONOutputConverter.getString(output)
livy_log = livy_log.json()["log"]
printed_output = analyzelog.get_output(livy_log)
dml.update_executed_file_wsession(execid, json.dumps(livy_log),
printed_output, sessionid, userid)
output["status"] = "success"
output["result"] = dict()
output["result"]["print"] = printed_output
output["result"]["id"] = execid
output["result"]["state"] = status_output.json()["state"]
output["result"]["log"] = livy_log
output["result"]["logfile"] = get_last_updated_file(
config.spark_events_path)
return JSONOutputConverter.getString(output)
def upload_file():
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
print('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
print('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
#file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
dml = DataManager.OutputDataManager(connection)
output = dml.insert_new_file(filename, file.stream.getvalue(),
userid)
return JSONOutputConverter.getString(output)
return ""
def get_output(execid):
"""
get the status
:return:
"""
output = dict()
dml = DataManager.OutputDataManager(connection)
sessionid = None
# get the session id from execution details
exec_output = dml.get_exec_details(execid)
if exec_output["status"] == "success":
output["status"] = "success"
output["result"] = dict()
else:
output["status"] = "fail"
output["result"] = dict()
output["result"]["message"] = "Session id not found"
return JSONOutputConverter.getString(output)
def download_output(execid):
"""
get the status
:return:
"""
output = dict()
dml = DataManager.OutputDataManager(connection)
sessionid = None
# get the session id from execution details
exec_output = dml.get_exec_details(execid)
if exec_output["status"] == "success":
filename = exec_output["result"]["fileid"]
sessionid = exec_output["result"]["sessionid"]
arguments = exec_output["result"]["arguments"]
output_folder = os.path.join(config.data_file_path, str(execid))
if os.path.exists(output_folder):
output_file_path = os.path.join(config.data_file_path,
"output" + str(datetime.now()))
shutil.make_archive(output_file_path, 'zip', output_folder)
return send_file(output_file_path + ".zip")
def upload_data_file():
if request.method == 'POST':
# check if the post request has the file part
if 'datafile' not in request.files:
print('No file part')
return redirect(request.url)
file = request.files['datafile']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
print('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
dml = DataManager.OutputDataManager(connection)
output = dml.insert_new_data_file(filename, config.data_file_path,
userid)
fileid = output["result"]["id"]
file.save(os.path.join(config.data_file_path, str(fileid)))
return JSONOutputConverter.getString(output)
return ""
def execute_file(fileid):
"""
Handles any page in the template folder
:return:
"""
dml = DataManager.OutputDataManager(connection)
arguments = None
if request.method == 'POST':
# Insert into db to get unique execution id
exec_output = dml.insert_executed_file(fileid, "", "", "", -1, userid)
exec_id = exec_output["result"]["id"]
rdata = request.data
args_list = []
# Parse all the arguments
if rdata is not None and len(rdata.strip()) > 1:
data_file_path = config.data_file_path
data_dict = json.loads(rdata)
args = data_dict["args"]
args = sorted(args, key=lambda x: int(x["seq"]))
for arg in args:
if arg["type"] == "file":
args_list.append(
os.path.join(data_file_path, arg["argument"]))
elif arg["type"] == "output":
args_list.append(
os.path.join(data_file_path, str(exec_id),
arg["argument"]))
else:
args_list.append(arg["argument"])
# get the python file from database and put it into file system
output = dml.get_file(fileid)
filecontent = output["result"]["content"]
filename = output["result"]["filename"]
file_path = app.config['UPLOAD_FOLDER'] + "/" + str(fileid) + ".py"
text_file = open(file_path, "w")
text_file.write(filecontent)
text_file.close()
# execute python file using livy
livy_output = livyquery.execute_file(filelocation=file_path,
arguments=args_list)
livy_output = livy_output.json()
sessionid = livy_output["id"]
#time.sleep(1)
livy_log = livyquery.log_status(sessionid).json()
livy_log = livy_log["log"]
job_status = livyquery.job_status(sessionid).json()
printed_output = analyzelog.get_output(livy_log)
exec_output = dml.update_executed_file_wsession(exec_id,
json.dumps(livy_log),
printed_output, sessionid,
userid)
# format output
output = dict()
output["status"] = "success"
output["result"] = dict()
output["result"]["print"] = printed_output
output["result"]["id"] = exec_id
output["result"]["state"] = job_status["state"]
return JSONOutputConverter.getString(output)
from sparkserve.datamanagement import DataManager
import config
connection = dict()
connection["file"] = config.sqlite_file
dm = DataManager.OutputDataManager(connection)
def test_insert_new_file():
"""
Test all necessary table creation
:return:
"""
filename = "testsqlite.py"
filecontent = "import test"
userid = 0
print(dm.insert_new_file(filename, filecontent, userid))
def test_update_file():
"""
Test all necessary table creation
:return:
"""
fileid = 4
filecontent = "import testupdated"
print(dm.update_file(fileid, filecontent))