class CloserAtlasPlugin: # constructor def __init__(self,job,datasets,log): self.jobSpec = job self.datasets = datasets self.tmpLog = LogWrapper(log,"{0} CloserAtlasPlugin".format(self.jobSpec.PandaID)) # execute def execute(self): try: # only for production if not self.jobSpec.prodSourceLabel in ['managed','test']: return True # only for urgent or high prio if not self.jobSpec.processingType in ['urgent'] and self.jobSpec.currentPriority <= 1000: return True # close datasets for datasetSpec in self.datasets: if re.search('_sub\d+$',datasetSpec.name) is None: continue if datasetSpec.status != 'tobeclosed': continue try: self.tmpLog.debug('immediate close {0}'.format(datasetSpec.name)) rucioAPI.closeDataset(datasetSpec.name) except Exception: errtype,errvalue = sys.exc_info()[:2] self.tmpLog.warning('failed to close : {0} {1}'.format(errtype,errvalue)) except Exception: errtype,errvalue = sys.exc_info()[:2] self.tmpLog.warning('failed to execute : {0} {1}'.format(errtype,errvalue)) return True
def application(environ, start_response): # get method name methodName = '' if 'SCRIPT_NAME' in environ: methodName = environ['SCRIPT_NAME'].split('/')[-1] tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName), seeMem=True) cont_length = int(environ.get('CONTENT_LENGTH', 0)) json_body = environ.get('CONTENT_TYPE', None) == 'application/json' tmpLog.debug("start content-length={} json={}".format(cont_length, json_body)) regStart = datetime.datetime.utcnow() retType = None # check method name if methodName not in allowedMethods: tmpLog.error("is forbidden") exeRes = "False : %s is forbidden" % methodName else: # get method object tmpMethod = None try: tmpMethod = globals()[methodName] except Exception: pass # object not found if tmpMethod is None: tmpLog.error("is undefined") exeRes = "False" else: body = b'' try: # dummy request object dummyReq = DummyReq(environ, tmpLog) if not dummyReq.authenticated: start_response('403 Forbidden', [('Content-Type', 'text/plain')]) return ["ERROR : Token authentication failed on the server side. {}".format( dummyReq.message).encode()] username = dummyReq.subprocess_env.get('SSL_CLIENT_S_DN', None) if username: username = CoreUtils.clean_user_id(username) if username in ban_user_list: errMsg = '{} is banned'.format(username) tmpLog.warning(errMsg) start_response('403 Forbidden', [('Content-Type', 'text/plain')]) return ["ERROR : {}".format(errMsg).encode()] # read contents while cont_length > 0: chunk = environ['wsgi.input'].read(min(cont_length, 1024*1024)) if not chunk: break cont_length -= len(chunk) body += chunk if cont_length > 0: raise OSError('partial read from client. {} bytes remaining'.format(cont_length)) if not json_body: # query string environ['wsgi.input'] = io.BytesIO(body) # get params tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ, keep_blank_values=1) # convert to map params = {} for tmpKey in list(tmpPars): if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None: # file params[tmpKey] = tmpPars[tmpKey] else: # string params[tmpKey] = tmpPars.getfirst(tmpKey) else: # json body = gzip.decompress(body) params = json.loads(body) if panda_config.entryVerbose: tmpLog.debug("with %s" % str(list(params))) param_list = [dummyReq] # exec exeRes = tmpMethod(*param_list, **params) # extract return type if isinstance(exeRes, dict): retType = exeRes['type'] exeRes = exeRes['content'] # convert bool to string if exeRes in [True,False]: exeRes = str(exeRes) except Exception as e: tmpLog.error("execution failure : {0}\n {1}".format(str(e), traceback.format_exc())) if hasattr(panda_config, 'dumpBadRequest') and panda_config.dumpBadRequest: try: with tempfile.NamedTemporaryFile(delete=False, prefix='req_dump_') as f: environ['WSGI_INPUT_DUMP'] = f.name f.write(body) os.chmod(f.name, 0o775) except Exception: tmpLog.error(traceback.format_exc()) pass errStr = "" for tmpKey in environ: tmpVal = environ[tmpKey] errStr += "%s : %s\n" % (tmpKey,str(tmpVal)) tmpLog.error(errStr) # return internal server error start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')]) # force kill to release memory if type(e) == OSError: tmpLog.warning('force restart due') os.kill(os.getpid(), signal.SIGINT) return [str(e).encode()] if panda_config.entryVerbose: tmpLog.debug("done") regTime = datetime.datetime.utcnow() - regStart tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds, regTime.microseconds/1000, len(str(exeRes)))) # return if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound: start_response('404 Not Found', [('Content-Type', 'text/plain')]) return ['not found'.encode()] elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect): start_response('302 Redirect', [('Location', exeRes.url)]) return ['redirect'.encode()] else: if retType == 'json': start_response('200 OK', [('Content-Type', 'application/json')]) else: start_response('200 OK', [('Content-Type', 'text/plain')]) if isinstance(exeRes, str): exeRes = exeRes.encode() return [exeRes]