def run_etl_process():
# ***
# Possible rpy2 code to run ETL R-script
# ***
try:
fin = flsk.request.args.get('i')
fout = flsk.request.args.get('o')
# appkey = flsk.request.args.get('appkey')
# key = flsk.request.args.get('key')
except Exception as e:
print("Exception", e)
return apilog.API_Response(
appkey="", key="", status=apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
import subprocess
command = 'Rscript' #or path to Rscript if required
path2script = '/media/Joe-Jalal/API/test_update3/test.R' #replace with actual path to script
args = [fin, fout]
args = [
'/media/Joe-Jalal/API/test_update3/tst_src3/',
'/media/Joe-Jalal/API/test_update3/Rscript_output/'
]
#replace with full path to input and output dirs
cmd = [command, path2script] + args
appkey, key = login_apptudio()
flog = apilog.API_logfname()
# apilog.LogStart(flog, 'etl', folder=True, PID=False)
apilog.LogStart(flog,
'etl',
infile=args[0],
outfile=args[1],
folder=True,
PID=False)
global x
x = subprocess.check_output(cmd, universal_newlines=True)
#x = subprocess.call(cmd)
print(x)
print(x.split()[1])
exit_etl_status = x.split()[1].split(':')[0]
print(exit_etl_status)
# apilog.Log(flog, 'Finish')
if exit_etl_status == 'EE000':
apilog.Log(flog, "Exit;ok", -1, -1, "INFO", apilog.EXIT_OK)
else: # if some error
apilog.Log(flog, "Exit;error", 0, -100, "ERROR", exit_etl_status)
# return flsk.jsonify(x)
return apilog.API_Response(appkey, key, flog, apilog.STATUS_OK, "",
"etl started", {})
def deepcrush_process():
log = {}
appkey = None
key = None
flog = None
try:
appkey = flsk.request.args.get('appkey')
key = flsk.request.args.get('key')
flog = flsk.request.args.get('log')
task = flsk.request.args.get('task')
print("deepcrush_process ", appkey, key)
except Exception as e:
print("Exception", e)
return apilog.API_Response(
appkey, key, flog, apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
return apilog.API_process(appkey, key, flog, task)
def varsig_process():
log = {}
appkey = None
key = None
flog = None
try:
appkey = flsk.request.args.get('appkey')
key = flsk.request.args.get('key')
flog = flsk.request.args.get('log')
task = flsk.request.args.get('task')
print(" varsig_process ", appkey, key)
except Exception as e:
print("Exception", e)
# log['status'] = apilog.URL_ERROR #'error'
# log['error'] = 'wrong url'
# log['exception'] = str(e)
# return apilog.API_Response(appkey, key, flog, apilog.URL_ERROR, log=log) # Flask returns json of map_data; either through curl or web browser.
return apilog.API_Response(
appkey, key, flog, apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
return apilog.API_process(appkey, key, flog, task)
def run_dpc_process():
sig = 0 #limit = accuracy = 0
try:
fin = flsk.request.args.get('i')
fout = flsk.request.args.get('o')
method = flsk.request.args.get('m') # method
sig = int(flsk.request.args.get('s')) # signif features
# appkey = flsk.request.args.get('appkey')
# key = flsk.request.args.get('key')
# limit = int(flsk.request.args.get('l')) #limit
# accuracy = int(flsk.request.args.get('a')) # accuracy
except Exception as e:
print("Exception", e)
# return apilog.API_Response(appkey="",key="",status="error", error="wrong url", result=str(e)) # Flask returns json of map_data; either through curl or web browser.
# return apilog.API_Response(appkey="",key="", status="EG100") # Flask returns json of map_data; either through curl or web browser.
return apilog.API_Response(
appkey="", key="", status=apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
# threshold = int(flsk.request.args.get('threshold')) # Grabbing value from 'threshold' key in Query String.
# scoop_size = int(flsk.request.args.get('scoop_size')) # Grabbing value from 'scoop_size' key in Query String.
# file_name = flsk.request.args.get('file_name') # Grabbing value from 'file_name' key in Query String.
print(fin, fout, method, sig)
# check valid arguments
errurl = vs.API_commandline(fin, fout, method, sig)
if errurl != '':
print(
'Invalid args given in uri.'
) # Error message for if values for threshold and scoop_size are invalid.
# return apilog.API_Response(appkey="",key="",status="error", error="wrong url", result=errurl) # Flask returns json of map_data; either through curl or web browser.
return apilog.API_Response(
appkey="", key="", flog="", status=errurl
) # Flask returns json of map_data; either through curl or web browser.
else:
# global data_
# data_ = dpcon.main_process(file_name, threshold, scoop_size) # Calls main_process in DpcController.py (DeepCrush's controller module).
print('Main process initiated via API')
# global map_data
# map_data = dpcon.maps_to_dict(data_) # Creates the dictionary structure to be stored in Apptudio.
# print('map_data:\n{}'.format(map_data)) # Printing structure; debugging.
appkey, key = login_apptudio(
) # Perform login to Apptudio; should return appkey and key.
print('appkey: {}, key: {}'.format(appkey, key))
data_id = post_api(
appkey, key, 0, map_data
) # POST dictionary structure to Apptudio, using appkey and key as authentication. 0 is process_id for now. Should return data_id if successful.
print('data_id: {}'.format(data_id))
r_json = get_api(
appkey, key, data_id
) # GET from Apptudio end; using appkey, key, and data_id to confirm map_data has been stored. Should return json response including map_data (map_count, map_0, map_1, ..., map_M).
print('r_json:\n{}'.format(r_json))
flog = apilog.API_logfname()
try:
varsigparam = vs.API_convertmethod(fin, fout, method, sig, appkey,
key, flog)
# varsigparam= "-i LungFull15.csv -o lung -m s+f -l 20000 -a 99 -s 4 --log --appkey %s --apikey %s" % (appkey, key)
# varsigparam= "-i AcceraPlacebo.csv -o accout -m s+f -l 20000 -a 99 -s 4 --log --appkey %s --apikey %s" % (appkey, key)
pid = vs.VarsigScriptProcess(varsigparam) # MAIN PROCESS
# pid= vs.VarsigScriptProcess(vs.VarsigScript, varsigparam) # MAIN PROCESS
# pid= vs.VarsigScriptProcessTest(vs.VarsigTest, "A", "DD" ) # MAIN PROCESS
print("PID", pid)
# vs.VarsigScript(varsigparam)
# vs.VarsigScript("-i LungFull15.csv -o lung.out -s 4")
except Exception as e:
print("Exception", e)
# return apilog.API_Response(appkey, key, status="error",error="cannot start varsig",result=str(e))
# return apilog.API_Response(appkey, key, flog, status="EG400")
return apilog.API_Response(appkey,
key,
flog,
status=apilog.PROCESS_ERR)
data = {}
data['appkey'] = appkey
data['key'] = key
data['status'] = "ok"
log = {}
log['time'] = "00:11:45"
log['process'] = "mixt method"
# return json.dump(("appkey", appkey),("key",key))
# return varsig_response(appkey, key, 'log') # Flask returns json of map_data; either through curl or web browser.
# return flsk.jsonify(("appkey",appkey), ("key",key), ("status","ok"), ("result","start process")) # Flask returns json of map_data; either through curl or web browser.
# return flsk.jsonify( data)#("appkey",appkey), ("key",key), ("status","ok"), ("result","start process")) # Flask returns json of map_data; either through curl or web browser.
# return apilog.API_Response(appkey, key, flog, "EG000","","varsig started", {})
return apilog.API_Response(appkey, key, flog, apilog.STATUS_OK, "",
"varsig started", {})
def run_dpc_process():
try:
threshold = int(
flsk.request.args.get('threshold')
) # Grabbing value from 'threshold' key in Query String.
scoop_size = int(
flsk.request.args.get('scoop_size')
) # Grabbing value from 'scoop_size' key in Query String.
file_name = flsk.request.args.get(
'file_name'
) # Grabbing value from 'file_name' key in Query String.
print(threshold, scoop_size, file_name)
print(type(threshold), type(scoop_size),
type(file_name)) # Print statements for debugging on my end.
except Exception as e:
print('Exception', e)
return apilog.API_Response(
appkey="", key="", status=apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
if threshold == 0 or scoop_size < 16:
print(
'Invalid args given in uri.'
) # Error message for if values for threshold and scoop_size are invalid.
return apilog.API_Response(
appkey="", key="", status=apilog.URL_ERROR
) # Flask returns json of map_data; either through curl or web browser.
else:
appkey, key = login_apptudio(
) # Perform login to Apptudio; should return appkey and key.
print('appkey: {}, key: {}'.format(appkey, key))
flog = apilog.API_logfname()
global data_
try:
pid = dpcon.main_process_pid(
file_name, threshold, scoop_size, appkey, key, flog
) # Calls main_process in DpcController.py (DeepCrush's controller module).
print('Main process initiated via API')
return apilog.API_Response(appkey, key, flog, apilog.STATUS_OK, "",
"deepcrush started", {})
except Exception as e:
print('Exception', e)
return apilog.API_Response(appkey,
key,
flog,
status=apilog.PROCESS_ERR)
# SCRIPT NEVER BE HERE
data_ = dpcon.load_data(appkey, key)
global map_data
map_data = dpcon.maps_to_dict(
data_
) # Creates the dictionary structure to be stored in Apptudio.
print(
'map_data:\n{}'.format(map_data)) # Printing structure; debugging.
data_id = post_api(
appkey, key, 0, map_data
) # POST dictionary structure to Apptudio, using appkey and key as authentication. 0 is process_id for now. Should return data_id if successful.
print('data_id: {}'.format(data_id))
r_json = get_api(
appkey, key, data_id
) # GET from Apptudio end; using appkey, key, and data_id to confirm map_data has been stored. Should return json response including map_data (map_count, map_0, map_1, ..., map_M).
print('r_json:\n{}'.format(r_json))
return apilog.API_Response(appkey, key, flog, apilog.EXIT_OK, "",
"deepcrush finished", {})