class CloserAtlasPlugin:
# constructor
def __init__(self,job,datasets,log):
self.jobSpec = job
self.datasets = datasets
self.tmpLog = LogWrapper(log,"{0} CloserAtlasPlugin".format(self.jobSpec.PandaID))
# execute
def execute(self):
try:
# only for production
if not self.jobSpec.prodSourceLabel in ['managed','test']:
return True
# only for urgent or high prio
if not self.jobSpec.processingType in ['urgent'] and self.jobSpec.currentPriority <= 1000:
return True
# close datasets
for datasetSpec in self.datasets:
if re.search('_sub\d+$',datasetSpec.name) is None:
continue
if datasetSpec.status != 'tobeclosed':
continue
try:
self.tmpLog.debug('immediate close {0}'.format(datasetSpec.name))
rucioAPI.closeDataset(datasetSpec.name)
except Exception:
errtype,errvalue = sys.exc_info()[:2]
self.tmpLog.warning('failed to close : {0} {1}'.format(errtype,errvalue))
except Exception:
errtype,errvalue = sys.exc_info()[:2]
self.tmpLog.warning('failed to execute : {0} {1}'.format(errtype,errvalue))
return True
def application(environ, start_response):
# get method name
methodName = ''
if 'SCRIPT_NAME' in environ:
methodName = environ['SCRIPT_NAME'].split('/')[-1]
tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName), seeMem=True)
cont_length = int(environ.get('CONTENT_LENGTH', 0))
json_body = environ.get('CONTENT_TYPE', None) == 'application/json'
tmpLog.debug("start content-length={} json={}".format(cont_length, json_body))
regStart = datetime.datetime.utcnow()
retType = None
# check method name
if methodName not in allowedMethods:
tmpLog.error("is forbidden")
exeRes = "False : %s is forbidden" % methodName
else:
# get method object
tmpMethod = None
try:
tmpMethod = globals()[methodName]
except Exception:
pass
# object not found
if tmpMethod is None:
tmpLog.error("is undefined")
exeRes = "False"
else:
body = b''
try:
# dummy request object
dummyReq = DummyReq(environ, tmpLog)
if not dummyReq.authenticated:
start_response('403 Forbidden', [('Content-Type', 'text/plain')])
return ["ERROR : Token authentication failed on the server side. {}".format(
dummyReq.message).encode()]
username = dummyReq.subprocess_env.get('SSL_CLIENT_S_DN', None)
if username:
username = CoreUtils.clean_user_id(username)
if username in ban_user_list:
errMsg = '{} is banned'.format(username)
tmpLog.warning(errMsg)
start_response('403 Forbidden', [('Content-Type', 'text/plain')])
return ["ERROR : {}".format(errMsg).encode()]
# read contents
while cont_length > 0:
chunk = environ['wsgi.input'].read(min(cont_length, 1024*1024))
if not chunk:
break
cont_length -= len(chunk)
body += chunk
if cont_length > 0:
raise OSError('partial read from client. {} bytes remaining'.format(cont_length))
if not json_body:
# query string
environ['wsgi.input'] = io.BytesIO(body)
# get params
tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ,
keep_blank_values=1)
# convert to map
params = {}
for tmpKey in list(tmpPars):
if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None:
# file
params[tmpKey] = tmpPars[tmpKey]
else:
# string
params[tmpKey] = tmpPars.getfirst(tmpKey)
else:
# json
body = gzip.decompress(body)
params = json.loads(body)
if panda_config.entryVerbose:
tmpLog.debug("with %s" % str(list(params)))
param_list = [dummyReq]
# exec
exeRes = tmpMethod(*param_list, **params)
# extract return type
if isinstance(exeRes, dict):
retType = exeRes['type']
exeRes = exeRes['content']
# convert bool to string
if exeRes in [True,False]:
exeRes = str(exeRes)
except Exception as e:
tmpLog.error("execution failure : {0}\n {1}".format(str(e), traceback.format_exc()))
if hasattr(panda_config, 'dumpBadRequest') and panda_config.dumpBadRequest:
try:
with tempfile.NamedTemporaryFile(delete=False, prefix='req_dump_') as f:
environ['WSGI_INPUT_DUMP'] = f.name
f.write(body)
os.chmod(f.name, 0o775)
except Exception:
tmpLog.error(traceback.format_exc())
pass
errStr = ""
for tmpKey in environ:
tmpVal = environ[tmpKey]
errStr += "%s : %s\n" % (tmpKey,str(tmpVal))
tmpLog.error(errStr)
# return internal server error
start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')])
# force kill to release memory
if type(e) == OSError:
tmpLog.warning('force restart due')
os.kill(os.getpid(), signal.SIGINT)
return [str(e).encode()]
if panda_config.entryVerbose:
tmpLog.debug("done")
regTime = datetime.datetime.utcnow() - regStart
tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds,
regTime.microseconds/1000,
len(str(exeRes))))
# return
if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound:
start_response('404 Not Found', [('Content-Type', 'text/plain')])
return ['not found'.encode()]
elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect):
start_response('302 Redirect', [('Location', exeRes.url)])
return ['redirect'.encode()]
else:
if retType == 'json':
start_response('200 OK', [('Content-Type', 'application/json')])
else:
start_response('200 OK', [('Content-Type', 'text/plain')])
if isinstance(exeRes, str):
exeRes = exeRes.encode()
return [exeRes]