def checkProxy(self, user_dn, production=False, role=None, name=None):
log_stream = LogWrapper(_logger,
'< name="{}" role={} >'.format(name, role))
log_stream.info('check proxy for {}'.format(user_dn))
"""Check the validity of a proxy."""
if role is not None:
tmpExtension = self.getExtension(role)
proxy_path = os.path.join(
self.__target_path,
str(hashlib.sha1(six.b(user_dn + tmpExtension)).hexdigest()))
elif production:
proxy_path = os.path.join(
self.__target_path,
str(hashlib.sha1(six.b(user_dn + '.prod')).hexdigest()))
else:
proxy_path = os.path.join(self.__target_path,
hashlib.sha1(six.b(user_dn)).hexdigest())
isOK = False
if os.path.isfile(proxy_path):
log_stream.info('proxy is there. Need to check validity')
cmd = "voms-proxy-info -exists -hours 94 -file %s" % proxy_path
stdout, stderr, status = execute(cmd, log_stream)
if stdout:
log_stream.info('stdout is %s ' % stdout)
if stderr:
log_stream.info('stderr is %s ' % stderr)
if status == 1:
log_stream.info(
'proxy expires in 94h or less. We need to renew proxy!')
ret = self.store(user_dn,
self.__cred_name,
production,
role=role,
log_stream=log_stream)
if ret == 0:
log_stream.info('proxy retrieval successful')
isOK = True
elif ret == 2:
log_stream.info('proxy retrieval on hold')
else:
log_stream.error('proxy retrieval failed')
else:
log_stream.info('proxy is valid for more than 3 days')
isOK = True
else:
log_stream.info(
'proxy is not in the cache repo. will try to get it from myproxy'
)
ret = self.store(user_dn,
self.__cred_name,
production,
role=role,
log_stream=log_stream)
if ret == 0:
log_stream.info('proxy stored successfully')
isOK = True
elif ret == 2:
log_stream.info('proxy retrieval on hold')
else:
log_stream.error('proxy retrieval failed')
if isOK:
plain_path = os.path.join(
self.__target_path,
hashlib.sha1(six.b(user_dn + '.plain')).hexdigest())
if os.path.isfile(plain_path):
return self.checkValidity(plain_path, log_stream)
else:
log_stream.error('plain proxy not there at the moment!')
def application(environ, start_response):
# get method name
methodName = ''
if 'SCRIPT_NAME' in environ:
methodName = environ['SCRIPT_NAME'].split('/')[-1]
tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName))
tmpLog.debug("start")
regStart = datetime.datetime.utcnow()
retType = None
# check method name
if not methodName in allowedMethods:
tmpLog.error("is forbidden")
exeRes = "False : %s is forbidden" % methodName
else:
# get method object
tmpMethod = None
try:
tmpMethod = globals()[methodName]
except Exception:
pass
# object not found
if tmpMethod is None:
tmpLog.error("is undefined")
exeRes = "False"
else:
try:
# get params
tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ,
keep_blank_values=1)
# convert to map
params = {}
for tmpKey in list(tmpPars):
if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None:
# file
params[tmpKey] = tmpPars[tmpKey]
else:
# string
params[tmpKey] = tmpPars.getfirst(tmpKey)
if panda_config.entryVerbose:
tmpLog.debug("with %s" % str(list(params)))
# dummy request object
dummyReq = DummyReq(environ, tmpLog)
param_list = [dummyReq]
# exec
exeRes = tmpMethod(*param_list, **params)
# extract return type
if isinstance(exeRes, dict):
retType = exeRes['type']
exeRes = exeRes['content']
# convert bool to string
if exeRes in [True,False]:
exeRes = str(exeRes)
except Exception as e:
tmpLog.error("execution failure : {0}".format(str(e)))
errStr = ""
for tmpKey in environ:
tmpVal = environ[tmpKey]
errStr += "%s : %s\n" % (tmpKey,str(tmpVal))
tmpLog.error(errStr)
# return internal server error
start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')])
return [str(e)]
if panda_config.entryVerbose:
tmpLog.debug("done")
regTime = datetime.datetime.utcnow() - regStart
tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds,
regTime.microseconds/1000,
len(str(exeRes))))
# return
if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound:
start_response('404 Not Found', [('Content-Type', 'text/plain')])
return ['not found']
elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect):
start_response('302 Redirect', [('Location', exeRes.url)])
return ['redirect']
else:
if retType == 'json':
start_response('200 OK', [('Content-Type', 'application/json')])
else:
start_response('200 OK', [('Content-Type', 'text/plain')])
if isinstance(exeRes, str):
exeRes = exeRes.encode()
return [exeRes]
def core_exec(sandbox_url, log_token, dump_workflow, ops_file, user_name, test_mode):
tmpLog = LogWrapper(_logger, log_token)
is_OK = True
is_fatal = False
request_id = None
if dump_workflow == 'True':
dump_workflow = True
else:
dump_workflow = False
if test_mode == 'True':
test_mode = True
else:
test_mode = False
try:
with open(ops_file) as f:
ops = json.load(f)
try:
os.remove(ops_file)
except Exception:
pass
# go to temp dir
cur_dir = os.getcwd()
with tempfile.TemporaryDirectory() as tmp_dirname:
os.chdir(tmp_dirname)
# download sandbox
tmpLog.info('downloading sandbox from {}'.format(sandbox_url))
with requests.get(sandbox_url, allow_redirects=True, verify=False, stream=True) as r:
if r.status_code == 400:
tmpLog.error("not found")
is_fatal = True
is_OK = False
elif r.status_code != 200:
tmpLog.error("bad HTTP response {}".format(r.status_code))
is_OK = False
# extract sandbox
if is_OK:
with open(ops['data']['sandbox'], 'wb') as fs:
for chunk in r.raw.stream(1024, decode_content=False):
if chunk:
fs.write(chunk)
fs.close()
tmp_stat, tmp_out = commands_get_status_output(
'tar xvfz {}'.format(ops['data']['sandbox']))
if tmp_stat != 0:
tmpLog.error(tmp_out)
dump_str = 'failed to extract {}'.format(ops['data']['sandbox'])
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
# parse workflow files
if is_OK:
tmpLog.info('parse workflow')
if ops['data']['language'] == 'cwl':
nodes, root_in = pcwl_utils.parse_workflow_file(ops['data']['workflowSpecFile'],
tmpLog)
with open(ops['data']['workflowInputFile']) as workflow_input:
data = yaml.safe_load(workflow_input)
s_id, t_nodes, nodes = pcwl_utils.resolve_nodes(nodes, root_in, data, 0, set(),
ops['data']['outDS'], tmpLog)
workflow_utils.set_workflow_outputs(nodes)
id_node_map = workflow_utils.get_node_id_map(nodes)
[node.resolve_params(ops['data']['taskParams'], id_node_map) for node in nodes]
dump_str = "the description was internally converted as follows\n" \
+ workflow_utils.dump_nodes(nodes)
tmpLog.info(dump_str)
for node in nodes:
s_check, o_check = node.verify()
tmp_str = 'Verification failure in ID:{} {}'.format(node.id, o_check)
if not s_check:
tmpLog.error(tmp_str)
dump_str += tmp_str
dump_str += '\n'
is_fatal = True
is_OK = False
else:
dump_str = "{} is not supported to describe the workflow"
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
# convert to workflow
if is_OK:
workflow_to_submit, dump_str_list = workflow_utils.convert_nodes_to_workflow(nodes)
try:
if workflow_to_submit:
if not test_mode:
tmpLog.info('submit workflow')
wm = ClientManager(host=get_rest_host())
request_id = wm.submit(workflow_to_submit, username=user_name)
else:
dump_str = 'workflow is empty'
tmpLog.error(dump_str)
is_fatal = True
is_OK = False
except Exception as e:
dump_str = 'failed to submit the workflow with {}'.format(str(e))
tmpLog.error('{} {}'.format(dump_str, traceback.format_exc()))
if dump_workflow:
tmpLog.debug('\n' + ''.join(dump_str_list))
os.chdir(cur_dir)
except Exception as e:
is_OK = False
is_fatal = True
tmpLog.error("failed to run with {} {}".format(str(e), traceback.format_exc()))
with tempfile.NamedTemporaryFile(delete=False, mode='w') as tmp_json:
json.dump([is_OK, is_fatal, request_id, tmpLog.dumpToString()], tmp_json)
print(tmp_json.name)
sys.exit(0)
def process(self, file_name, to_delete=False, test_mode=False, get_log=False, dump_workflow=False):
try:
is_fatal = False
is_OK = True
request_id = None
dump_str = None
with open(file_name) as f:
ops = json.load(f)
user_name = clean_user_id(ops["userName"])
base_platform = ops['data'].get('base_platform')
for task_type in ops['data']['taskParams']:
ops['data']['taskParams'][task_type]['userName'] = user_name
if base_platform:
ops['data']['taskParams'][task_type]['basePlatform'] = base_platform
log_token = '< id="{}" test={} outDS={} >'.format(user_name, test_mode, ops['data']['outDS'])
tmpLog = LogWrapper(self.log, log_token)
tmpLog.info('start {}'.format(file_name))
sandbox_url = os.path.join(ops['data']['sourceURL'], 'cache', ops['data']['sandbox'])
# IO through json files
ops_file = tempfile.NamedTemporaryFile(delete=False, mode='w')
json.dump(ops, ops_file)
ops_file.close()
# execute main in another process to avoid chdir mess
tmp_stat, tmp_out = commands_get_status_output("python {} {} '{}' {} {} '{}' {}".format(
__file__, sandbox_url, log_token, dump_workflow, ops_file.name,
user_name, test_mode))
if tmp_stat:
is_OK = False
tmpLog.error('main execution failed with {}:{}'.format(tmp_stat, tmp_out))
else:
with open(tmp_out.split('\n')[-1]) as tmp_out_file:
is_OK, is_fatal, request_id, dump_str = json.load(tmp_out_file)
try:
os.remove(tmp_out)
except Exception:
pass
if not get_log:
if is_OK:
tmpLog.info('is_OK={} request_id={}'.format(is_OK, request_id))
else:
tmpLog.info('is_OK={} is_fatal={} request_id={}'.format(is_OK, is_fatal, request_id))
if to_delete or (not test_mode and (is_OK or is_fatal)):
dump_str = tmpLog.dumpToString() + dump_str
tmpLog.debug('delete {}'.format(file_name))
try:
os.remove(file_name)
except Exception:
pass
# send notification
if not test_mode and self.taskBuffer is not None:
toAdder = self.taskBuffer.getEmailAddr(user_name)
if toAdder is None or toAdder.startswith('notsend'):
tmpLog.debug('skip to send notification since suppressed')
else:
# message
if is_OK:
mailSubject = "PANDA Notification for Workflow {}".format(ops['data']['outDS'])
mailBody = "Hello,\n\nWorkflow:{} has been accepted with RequestID:{}\n\n".\
format(ops['data']['outDS'], request_id)
else:
mailSubject = "PANDA WARNING for Workflow={}".format(ops['data']['outDS'])
mailBody = "Hello,\n\nWorkflow {} was not accepted\n\n".\
format(ops['data']['outDS'], request_id)
mailBody += "Reason : %s\n" % dump_str
# send
tmpSM = MailUtils().send(toAdder, mailSubject, mailBody)
tmpLog.debug('sent message with {}'.format(tmpSM))
except Exception as e:
is_OK = False
tmpLog.error("failed to run with {} {}".format(str(e), traceback.format_exc()))
if get_log:
ret_val = {'status': is_OK}
if is_OK:
ret_val['log'] = dump_str
else:
if dump_str is None:
ret_val['log'] = tmpLog.dumpToString()
else:
ret_val['log'] = dump_str
return ret_val
def application(environ, start_response):
# get method name
methodName = ''
if 'SCRIPT_NAME' in environ:
methodName = environ['SCRIPT_NAME'].split('/')[-1]
tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName), seeMem=True)
cont_length = int(environ.get('CONTENT_LENGTH', 0))
json_body = environ.get('CONTENT_TYPE', None) == 'application/json'
tmpLog.debug("start content-length={} json={}".format(cont_length, json_body))
regStart = datetime.datetime.utcnow()
retType = None
# check method name
if methodName not in allowedMethods:
tmpLog.error("is forbidden")
exeRes = "False : %s is forbidden" % methodName
else:
# get method object
tmpMethod = None
try:
tmpMethod = globals()[methodName]
except Exception:
pass
# object not found
if tmpMethod is None:
tmpLog.error("is undefined")
exeRes = "False"
else:
body = b''
try:
# dummy request object
dummyReq = DummyReq(environ, tmpLog)
if not dummyReq.authenticated:
start_response('403 Forbidden', [('Content-Type', 'text/plain')])
return ["ERROR : Token authentication failed on the server side. {}".format(
dummyReq.message).encode()]
username = dummyReq.subprocess_env.get('SSL_CLIENT_S_DN', None)
if username:
username = CoreUtils.clean_user_id(username)
if username in ban_user_list:
errMsg = '{} is banned'.format(username)
tmpLog.warning(errMsg)
start_response('403 Forbidden', [('Content-Type', 'text/plain')])
return ["ERROR : {}".format(errMsg).encode()]
# read contents
while cont_length > 0:
chunk = environ['wsgi.input'].read(min(cont_length, 1024*1024))
if not chunk:
break
cont_length -= len(chunk)
body += chunk
if cont_length > 0:
raise OSError('partial read from client. {} bytes remaining'.format(cont_length))
if not json_body:
# query string
environ['wsgi.input'] = io.BytesIO(body)
# get params
tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ,
keep_blank_values=1)
# convert to map
params = {}
for tmpKey in list(tmpPars):
if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None:
# file
params[tmpKey] = tmpPars[tmpKey]
else:
# string
params[tmpKey] = tmpPars.getfirst(tmpKey)
else:
# json
body = gzip.decompress(body)
params = json.loads(body)
if panda_config.entryVerbose:
tmpLog.debug("with %s" % str(list(params)))
param_list = [dummyReq]
# exec
exeRes = tmpMethod(*param_list, **params)
# extract return type
if isinstance(exeRes, dict):
retType = exeRes['type']
exeRes = exeRes['content']
# convert bool to string
if exeRes in [True,False]:
exeRes = str(exeRes)
except Exception as e:
tmpLog.error("execution failure : {0}\n {1}".format(str(e), traceback.format_exc()))
if hasattr(panda_config, 'dumpBadRequest') and panda_config.dumpBadRequest:
try:
with tempfile.NamedTemporaryFile(delete=False, prefix='req_dump_') as f:
environ['WSGI_INPUT_DUMP'] = f.name
f.write(body)
os.chmod(f.name, 0o775)
except Exception:
tmpLog.error(traceback.format_exc())
pass
errStr = ""
for tmpKey in environ:
tmpVal = environ[tmpKey]
errStr += "%s : %s\n" % (tmpKey,str(tmpVal))
tmpLog.error(errStr)
# return internal server error
start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')])
# force kill to release memory
if type(e) == OSError:
tmpLog.warning('force restart due')
os.kill(os.getpid(), signal.SIGINT)
return [str(e).encode()]
if panda_config.entryVerbose:
tmpLog.debug("done")
regTime = datetime.datetime.utcnow() - regStart
tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds,
regTime.microseconds/1000,
len(str(exeRes))))
# return
if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound:
start_response('404 Not Found', [('Content-Type', 'text/plain')])
return ['not found'.encode()]
elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect):
start_response('302 Redirect', [('Location', exeRes.url)])
return ['redirect'.encode()]
else:
if retType == 'json':
start_response('200 OK', [('Content-Type', 'application/json')])
else:
start_response('200 OK', [('Content-Type', 'text/plain')])
if isinstance(exeRes, str):
exeRes = exeRes.encode()
return [exeRes]
