def __call__(self, *args, **kwargs): log = LogWrapper( _logger, 'pid={} thr={} {}'.format(os.getpid(), threading.current_thread().ident, self.methodName)) log.debug('start') # get lock among children i = self.childlock.get() # make dict to send it master self.commDict[i].update({ 'methodName': self.methodName, 'args': pickle.dumps(args), 'kwargs': pickle.dumps(kwargs) }) # send notification to master self.comLock[i].release() # wait response self.resLock[i].acquire() res = pickle.loads(self.commDict[i]['res']) statusCode = self.commDict[i]['stat'] # release lock to children self.childlock.put(i) log.debug('end') # return if statusCode == 0: return res else: errtype, errvalue = res raise RuntimeError("{0}: {1} {2}".format(self.methodName, errtype.__name__, errvalue))
def __init__(self, taskBuffer, jobID, jobStatus, attemptNr, ignoreTmpError=True, siteMapper=None, pid=None, prelock_pid=None, lock_offset=10): self.job = None self.jobID = jobID self.jobStatus = jobStatus self.taskBuffer = taskBuffer self.ignoreTmpError = ignoreTmpError self.lock_offset = lock_offset self.siteMapper = siteMapper self.datasetMap = {} self.extraInfo = { 'surl': {}, 'nevents': {}, 'lbnr': {}, 'endpoint': {}, 'guid': {} } self.attemptNr = attemptNr self.pid = pid self.prelock_pid = prelock_pid self.data = None # logger self.logger = LogWrapper(_logger, str(self.jobID))
def run(self): # get logger tmpLog = LogWrapper(_logger,'<vuid={0} site={1} name={2}>'.format(self.vuid, self.site, self.dataset)) # query dataset tmpLog.debug("start") if self.vuid is not None: dataset = self.taskBuffer.queryDatasetWithMap({'vuid':self.vuid}) else: dataset = self.taskBuffer.queryDatasetWithMap({'name':self.dataset}) if dataset is None: tmpLog.error("Not found") tmpLog.debug("end") return tmpLog.debug("type:%s name:%s" % (dataset.type,dataset.name)) if dataset.type == 'dispatch': # activate jobs in jobsDefined Activator(self.taskBuffer,dataset).start() if dataset.type == 'output': if dataset.name is not None and re.search('^panda\..*_zip$',dataset.name) is not None: # start unmerge jobs Activator(self.taskBuffer,dataset,enforce=True).start() else: # finish transferring jobs Finisher(self.taskBuffer,dataset,site=self.site).start() tmpLog.debug("end")
def put_file_recovery_request(req, jediTaskID, dryRun=None): if not Protocol.isSecure(req): return json.dumps((False, "ERROR : no HTTPS")) userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') tmpLog = LogWrapper(_logger, 'put_file_recovery_request < jediTaskID={}'.format(jediTaskID)) tmpLog.debug("start user={}".format(userName)) # get total size try: jediTaskID = int(jediTaskID) # make filename evpFileName = '%s/recov.%s' % (panda_config.cache_dir,str(uuid.uuid4())) tmpLog.debug("file={}".format(evpFileName)) # write with open(evpFileName, 'w') as fo: data = {"userName": userName, "creationTime": creationTime, "jediTaskID": int(jediTaskID) } if dryRun: data['dryRun'] = True json.dump(data, fo) except Exception as e: errStr = "cannot put request due to {} ".format(str(e)) tmpLog.error(errStr + traceback.format_exc()) return json.dumps((False, errStr)) tmpLog.debug('done') return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def main(tbuf=None, **kwargs): # logger tmpLog = LogWrapper(_logger) tmpLog.debug("================= start ==================") # instantiate TB if tbuf is None: from pandaserver.taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) else: taskBuffer = tbuf # instantiate MyProxy I/F my_proxy_interface_instance = panda_proxy_cache.MyProxyInterface() # roles if hasattr(panda_config,'proxy_cache_roles'): roles = panda_config.proxy_cache_roles.split(',') else: roles = ['atlas','atlas:/atlas/Role=production','atlas:/atlas/Role=pilot'] # get users sql = 'select distinct DN FROM ATLAS_PANDAMETA.users WHERE GRIDPREF LIKE :patt' varMap = {} varMap[':patt'] = '%p%' tmpStat,tmpRes = taskBuffer.querySQLS(sql,varMap) for realDN, in tmpRes: if realDN is None: continue realDN = CoreUtils.get_bare_dn(realDN, keep_digits=False) name = taskBuffer.cleanUserID(realDN) # check proxy tmpLog.debug("check proxy cache for {}".format(name)) for role in roles: my_proxy_interface_instance.checkProxy(realDN, role=role, name=name) tmpLog.debug("done")
def __init__(self, taskBuffer, jobID, jobStatus, xmlFile, ignoreTmpError=True, siteMapper=None): self.job = None self.jobID = jobID self.jobStatus = jobStatus self.taskBuffer = taskBuffer self.ignoreTmpError = ignoreTmpError self.lockXML = None self.siteMapper = siteMapper self.attemptNr = None self.xmlFile = xmlFile self.datasetMap = {} self.extraInfo = { 'surl': {}, 'nevents': {}, 'lbnr': {}, 'endpoint': {}, 'guid': {} } # exstract attemptNr try: tmpAttemptNr = self.xmlFile.split('/')[-1].split('_')[-1] if re.search('^\d+$', tmpAttemptNr) is not None: self.attemptNr = int(tmpAttemptNr) except Exception: pass # logger self.logger = LogWrapper(_logger, str(self.jobID))
def __init__(self, taskBuffer, siteMapper, evpFileName, ignoreError): self.taskBuffer = taskBuffer self.siteMapper = siteMapper self.ignoreError = ignoreError self.evpFileName = evpFileName self.token = datetime.datetime.utcnow().isoformat(' ') # logger self.logger = LogWrapper(_logger, self.token) self.pd2p = DynDataDistributer.DynDataDistributer([], self.taskBuffer, self.siteMapper, token=' ', logger=self.logger) self.userDatasetName = '' self.creationTime = '' self.params = '' self.lockedBy = '' self.evpFile = None self.userTaskName = '' # message buffer self.msgBuffer = [] self.lineLimit = 100 # JEDI self.jediTaskID = None self.prodSourceLabel = None self.job_label = None
def uploadLog(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename)) tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN'])) # size check sizeLimit = 100 * 1024 * 1024 # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except Exception: if "content-length" in req.headers_in: tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "failed to upload log due to size limit" tmpLog.error(errStr) tmpLog.debug("end") return errStr jediLogDir = '/jedilog' retStr = '' try: fileBaseName = file.filename.split('/')[-1] fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir, fileBaseName) # delete old file if os.path.exists(fileFullPath): os.remove(fileFullPath) # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() tmpLog.debug("written to {0}".format(fileFullPath)) retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None), jediLogDir, fileBaseName) except Exception: errtype, errvalue = sys.exc_info()[:2] errStr = "failed to write log with {0}:{1}".format( errtype.__name__, errvalue) tmpLog.error(errStr) tmpLog.debug("end") return errStr tmpLog.debug("end") return retStr
def delete_checkpoint(req, task_id, sub_id): tmpLog = LogWrapper(_logger, 'delete_checkpoint <jediTaskID={0} ID={1}>'.format(task_id, sub_id)) status = True if not Protocol.isSecure(req): msg = 'insecure request' tmpLog.error(msg) status = False else: tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN']) try: fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id)) os.remove(fileFullPath) msg = 'done' tmpLog.debug(msg) except Exception as e: msg = "failed to delete file due to {0}".format(str(e)) tmpLog.error(msg) status = False return json.dumps({'status': status, 'message': msg})
def put_checkpoint(req, file): tmpLog = LogWrapper(_logger, 'put_checkpoint <jediTaskID_subID={0}>'.format(file.filename)) status = False if not Protocol.isSecure(req): errStr = 'insecure request' tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN']) # extract taskID and subID try: task_id, sub_id = file.filename.split('/')[-1].split('_') except Exception: errStr = 'failed to extract ID' tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) # size check sizeLimit = 500 * 1024 * 1024 # get file size try: contentLength = long(req.headers_in["content-length"]) except Exception as e: errStr = "cannot get int(content-length) due to {0}".format(str(e)) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "exceeded size limit %s>%s" % (contentLength, sizeLimit) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) try: fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id)) # write with open(fileFullPath,'wb') as fo: fo.write(file.file.read()) except Exception as e: errStr = "cannot write file due to {0}".format(str(e)) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) status = True tmpMsg = "successfully placed at {0}".format(fileFullPath) tmpLog.debug(tmpMsg) return json.dumps({'status': status, 'message': tmpMsg})
def put_workflow_request(req, data, check=False): if not Protocol.isSecure(req): return json.dumps((False, "ERROR : no HTTPS")) userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') tmpLog = LogWrapper(_logger, 'put_workflow_request') tmpLog.debug("start user={} check={}".format(userName, check)) if check == 'True' or check is True: check = True else: check = False # get total size try: # make filename evpFileName = '%s/workflow.%s' % (panda_config.cache_dir,str(uuid.uuid4())) tmpLog.debug("file={}".format(evpFileName)) # write with open(evpFileName, 'w') as fo: data = {"userName": userName, "creationTime": creationTime, "data": json.loads(data), } json.dump(data, fo) # check if check: tmpLog.debug('checking') from pandaserver.taskbuffer.workflow_processor import WorkflowProcessor processor = WorkflowProcessor(log_stream=_logger) ret = processor.process(evpFileName, True, True, True, True) if os.path.exists(evpFileName): try: os.remove(evpFileName) except Exception: pass tmpLog.debug('done') return json.dumps((True, ret)) except Exception as e: errStr = "cannot put request due to {} ".format(str(e)) tmpLog.error(errStr + traceback.format_exc()) return json.dumps((False, errStr)) tmpLog.debug('done') return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def checkProxy(self, user_dn, production=False, role=None, name=None): log_stream = LogWrapper(_logger, '< name="{}" role={} >'.format(name, role)) log_stream.info('check proxy for {}'.format(user_dn)) """Check the validity of a proxy.""" if role is not None: tmpExtension = self.getExtension(role) proxy_path = os.path.join( self.__target_path, str(hashlib.sha1(six.b(user_dn + tmpExtension)).hexdigest())) elif production: proxy_path = os.path.join( self.__target_path, str(hashlib.sha1(six.b(user_dn + '.prod')).hexdigest())) else: proxy_path = os.path.join(self.__target_path, hashlib.sha1(six.b(user_dn)).hexdigest()) isOK = False if os.path.isfile(proxy_path): log_stream.info('proxy is there. Need to check validity') cmd = "voms-proxy-info -exists -hours 94 -file %s" % proxy_path stdout, stderr, status = execute(cmd, log_stream) if stdout: log_stream.info('stdout is %s ' % stdout) if stderr: log_stream.info('stderr is %s ' % stderr) if status == 1: log_stream.info( 'proxy expires in 94h or less. We need to renew proxy!') ret = self.store(user_dn, self.__cred_name, production, role=role, log_stream=log_stream) if ret == 0: log_stream.info('proxy retrieval successful') isOK = True elif ret == 2: log_stream.info('proxy retrieval on hold') else: log_stream.error('proxy retrieval failed') else: log_stream.info('proxy is valid for more than 3 days') isOK = True else: log_stream.info( 'proxy is not in the cache repo. will try to get it from myproxy' ) ret = self.store(user_dn, self.__cred_name, production, role=role, log_stream=log_stream) if ret == 0: log_stream.info('proxy stored successfully') isOK = True elif ret == 2: log_stream.info('proxy retrieval on hold') else: log_stream.error('proxy retrieval failed') if isOK: plain_path = os.path.join( self.__target_path, hashlib.sha1(six.b(user_dn + '.plain')).hexdigest()) if os.path.isfile(plain_path): return self.checkValidity(plain_path, log_stream) else: log_stream.error('plain proxy not there at the moment!')
def putFile(req, file): tmpLog = LogWrapper(_logger, 'putFile-{}'.format(datetime.datetime.utcnow().isoformat('/'))) if not Protocol.isSecure(req): tmpLog.error('No SSL_CLIENT_S_DN') return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False # user name username = CoreUtils.clean_user_id(req.subprocess_env['SSL_CLIENT_S_DN']) tmpLog.debug("start %s %s" % (username, file.filename)) # size check fullSizeLimit = 768*1024*1024 if not file.filename.startswith('sources.'): noBuild = True sizeLimit = 100*1024*1024 else: noBuild = False sizeLimit = fullSizeLimit # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except Exception: if "content-length" in req.headers_in: tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % (contentLength,sizeLimit) if noBuild: errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit" else: errStr += " Please remove redundant files from your workarea" tmpLog.error(errStr) tmpLog.debug("end") return errStr try: fileName = file.filename.split('/')[-1] fileFullPath = '%s/%s' % (panda_config.cache_dir, fileName) # avoid overwriting if os.path.exists(fileFullPath): # touch os.utime(fileFullPath,None) # send error message errStr = "ERROR : Cannot overwrite file" tmpLog.debug('cannot overwrite file %s' % fileName) tmpLog.debug("end") return errStr # write fo = open(fileFullPath,'wb') fileContent = file.file.read() if hasattr(panda_config, 'compress_file_names') and \ [True for patt in panda_config.compress_file_names.split(',') if re.search(patt, fileName) is not None]: fileContent = gzip.compress(fileContent) fo.write(fileContent) fo.close() except Exception: errStr = "ERROR : Cannot write file" tmpLog.error(errStr) tmpLog.debug("end") return errStr # checksum try: # decode Footer footer = fileContent[-8:] checkSum,isize = struct.unpack("II",footer) tmpLog.debug("CRC from gzip Footer %s" % checkSum) except Exception: # calculate on the fly """ import zlib checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF """ # use None to avoid delay for now checkSum = None tmpLog.debug("CRC calculated %s" % checkSum) # file size fileSize = len(fileContent) tmpLog.debug("written dn=%s file=%s size=%s crc=%s" % \ (username, fileFullPath, fileSize, checkSum)) # put file info to DB if panda_config.record_sandbox_info: to_insert = True for patt in IGNORED_SUFFIX: if file.filename.endswith(patt): to_insert = False break if not to_insert: tmpLog.debug("skipped to insert to DB") else: statClient,outClient = Client.insertSandboxFileInfo(username,file.filename, fileSize,checkSum) if statClient != 0 or outClient.startswith("ERROR"): tmpLog.error("failed to put sandbox to DB with %s %s" % (statClient,outClient)) #_logger.debug("putFile : end") #return "ERROR : Cannot insert sandbox to DB" else: tmpLog.debug("inserted sandbox to DB with %s" % outClient) tmpLog.debug("end") return True
def main(argv=tuple(), tbuf=None, **kwargs): try: long except NameError: long = int tmpLog = LogWrapper(_logger, None) tmpLog.debug("===================== start =====================") # current minute currentMinute = datetime.datetime.utcnow().minute # instantiate TB if tbuf is None: from pandaserver.taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) else: taskBuffer = tbuf # instantiate sitemapper aSiteMapper = SiteMapper(taskBuffer) # delete tmpLog.debug("Del session") status, retSel = taskBuffer.querySQLS( "SELECT MAX(PandaID) FROM ATLAS_PANDA.jobsDefined4", {}) if retSel is not None: try: maxID = retSel[0][0] tmpLog.debug("maxID : %s" % maxID) if maxID is not None: varMap = {} varMap[':maxID'] = maxID varMap[':jobStatus1'] = 'activated' varMap[':jobStatus2'] = 'waiting' varMap[':jobStatus3'] = 'failed' varMap[':jobStatus4'] = 'cancelled' status, retDel = taskBuffer.querySQLS( "DELETE FROM ATLAS_PANDA.jobsDefined4 WHERE PandaID<:maxID AND jobStatus IN (:jobStatus1,:jobStatus2,:jobStatus3,:jobStatus4)", varMap) except Exception: pass # count # of getJob/updateJob in dispatcher's log try: # don't update when logrotate is running timeNow = datetime.datetime.utcnow() logRotateTime = timeNow.replace(hour=3, minute=2, second=0, microsecond=0) if (timeNow > logRotateTime and (timeNow-logRotateTime) < datetime.timedelta(minutes=5)) or \ (logRotateTime > timeNow and (logRotateTime-timeNow) < datetime.timedelta(minutes=5)): tmpLog.debug("skip pilotCounts session for logrotate") else: # log filename dispLogName = '%s/panda-PilotRequests.log' % panda_config.logdir # time limit timeLimit = datetime.datetime.utcnow() - datetime.timedelta( hours=3) timeLimitS = datetime.datetime.utcnow() - datetime.timedelta( hours=1) # check if tgz is required com = 'head -1 %s' % dispLogName lostat, loout = commands_get_status_output(com) useLogTgz = True if lostat == 0: match = re.search('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', loout) if match is not None: startTime = datetime.datetime(*time.strptime( match.group(0), '%Y-%m-%d %H:%M:%S')[:6]) # current log contains all info if startTime < timeLimit: useLogTgz = False # log files dispLogNameList = [dispLogName] if useLogTgz: today = datetime.date.today() dispLogNameList.append('{0}-{1}.gz'.format( dispLogName, today.strftime('%Y%m%d'))) # delete tmp commands_get_status_output('rm -f %s.tmp-*' % dispLogName) # tmp name tmpLogName = '%s.tmp-%s' % (dispLogName, datetime.datetime.utcnow( ).strftime('%Y-%m-%d-%H-%M-%S')) # loop over all files pilotCounts = {} pilotCountsS = {} for tmpDispLogName in dispLogNameList: # expand or copy if tmpDispLogName.endswith('.gz'): com = 'gunzip -c %s > %s' % (tmpDispLogName, tmpLogName) else: com = 'cp %s %s' % (tmpDispLogName, tmpLogName) lostat, loout = commands_get_status_output(com) if lostat != 0: errMsg = 'failed to expand/copy %s with : %s' % ( tmpDispLogName, loout) raise RuntimeError(errMsg) # search string sStr = '^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*' sStr += 'method=(.+),site=(.+),node=(.+),type=(.+)' # read logFH = open(tmpLogName) for line in logFH: # check format match = re.search(sStr, line) if match is not None: # check timerange timeStamp = datetime.datetime(*time.strptime( match.group(1), '%Y-%m-%d %H:%M:%S')[:6]) if timeStamp < timeLimit: continue tmpMethod = match.group(2) tmpSite = match.group(3) tmpNode = match.group(4) tmpType = match.group(5) # protection against corrupted entries from pilot, # e.g. pilot reading site json from cvmfs while it was being updated if tmpSite not in aSiteMapper.siteSpecList: continue # sum pilotCounts.setdefault(tmpSite, {}) pilotCounts[tmpSite].setdefault(tmpMethod, {}) pilotCounts[tmpSite][tmpMethod].setdefault(tmpNode, 0) pilotCounts[tmpSite][tmpMethod][tmpNode] += 1 # short if timeStamp > timeLimitS: if tmpSite not in pilotCountsS: pilotCountsS[tmpSite] = dict() if tmpMethod not in pilotCountsS[tmpSite]: pilotCountsS[tmpSite][tmpMethod] = dict() if tmpNode not in pilotCountsS[tmpSite][tmpMethod]: pilotCountsS[tmpSite][tmpMethod][tmpNode] = 0 pilotCountsS[tmpSite][tmpMethod][tmpNode] += 1 # close logFH.close() # delete tmp commands_get_status_output('rm %s' % tmpLogName) # update hostID = panda_config.pserverhost.split('.')[0] tmpLog.debug("pilotCounts session") retPC = taskBuffer.updateSiteData(hostID, pilotCounts, interval=3) tmpLog.debug(retPC) retPC = taskBuffer.updateSiteData(hostID, pilotCountsS, interval=1) tmpLog.debug(retPC) except Exception: errType, errValue = sys.exc_info()[:2] tmpLog.error("updateJob/getJob : %s %s" % (errType, errValue)) # nRunning tmpLog.debug("nRunning session") try: if (currentMinute / panda_config.nrun_interval ) % panda_config.nrun_hosts == panda_config.nrun_snum: retNR = taskBuffer.insertnRunningInSiteData() tmpLog.debug(retNR) except Exception: errType, errValue = sys.exc_info()[:2] tmpLog.error("nRunning : %s %s" % (errType, errValue)) # session for co-jumbo jobs tmpLog.debug("co-jumbo session") try: ret = taskBuffer.getCoJumboJobsToBeFinished(30, 0, 1000) if ret is None: tmpLog.debug("failed to get co-jumbo jobs to finish") else: coJumboA, coJumboD, coJumboW, coJumboTokill = ret tmpLog.debug("finish {0} co-jumbo jobs in Active".format( len(coJumboA))) if len(coJumboA) > 0: jobSpecs = taskBuffer.peekJobs(coJumboA, fromDefined=False, fromActive=True, fromArchived=False, fromWaiting=False) for jobSpec in jobSpecs: fileCheckInJEDI = taskBuffer.checkInputFileStatusInJEDI( jobSpec) if not fileCheckInJEDI: jobSpec.jobStatus = 'closed' jobSpec.jobSubStatus = 'cojumbo_wrong' jobSpec.taskBufferErrorCode = pandaserver.taskbuffer.ErrorCode.EC_EventServiceInconsistentIn taskBuffer.archiveJobs([jobSpec], False) tmpLog.debug("finish {0} co-jumbo jobs in Defined".format( len(coJumboD))) if len(coJumboD) > 0: jobSpecs = taskBuffer.peekJobs(coJumboD, fromDefined=True, fromActive=False, fromArchived=False, fromWaiting=False) for jobSpec in jobSpecs: fileCheckInJEDI = taskBuffer.checkInputFileStatusInJEDI( jobSpec) if not fileCheckInJEDI: jobSpec.jobStatus = 'closed' jobSpec.jobSubStatus = 'cojumbo_wrong' jobSpec.taskBufferErrorCode = pandaserver.taskbuffer.ErrorCode.EC_EventServiceInconsistentIn taskBuffer.archiveJobs([jobSpec], True) tmpLog.debug("finish {0} co-jumbo jobs in Waiting".format( len(coJumboW))) if len(coJumboW) > 0: jobSpecs = taskBuffer.peekJobs(coJumboW, fromDefined=False, fromActive=False, fromArchived=False, fromWaiting=True) for jobSpec in jobSpecs: fileCheckInJEDI = taskBuffer.checkInputFileStatusInJEDI( jobSpec) if not fileCheckInJEDI: jobSpec.jobStatus = 'closed' jobSpec.jobSubStatus = 'cojumbo_wrong' jobSpec.taskBufferErrorCode = pandaserver.taskbuffer.ErrorCode.EC_EventServiceInconsistentIn taskBuffer.archiveJobs([jobSpec], False, True) tmpLog.debug("kill {0} co-jumbo jobs in Waiting".format( len(coJumboTokill))) if len(coJumboTokill) > 0: jediJobs = list(coJumboTokill) nJob = 100 iJob = 0 while iJob < len(jediJobs): tmpLog.debug(' killing %s' % str(jediJobs[iJob:iJob + nJob])) Client.killJobs(jediJobs[iJob:iJob + nJob], 51, keepUnmerged=True) iJob += nJob except Exception: errStr = traceback.format_exc() tmpLog.error(errStr) tmpLog.debug("Fork session") # thread for fork class ForkThr(threading.Thread): def __init__(self, fileName): threading.Thread.__init__(self) self.fileName = fileName def run(self): if 'VIRTUAL_ENV' in os.environ: prefix = os.environ['VIRTUAL_ENV'] else: prefix = '' setupStr = 'source {0}/etc/sysconfig/panda_server; '.format(prefix) runStr = '%s/python -Wignore ' % panda_config.native_python runStr += panda_config.pandaPython_dir + '/dataservice/forkSetupper.py -i ' runStr += self.fileName if self.fileName.split('/')[-1].startswith('set.NULL.'): runStr += ' -t' comStr = setupStr + runStr tmpLog.debug(comStr) commands_get_status_output(comStr) # get set.* files filePatt = panda_config.logdir + '/' + 'set.*' fileList = glob.glob(filePatt) # the max number of threads maxThr = 10 nThr = 0 # loop over all files forkThrList = [] timeNow = datetime.datetime.utcnow() for tmpName in fileList: if not os.path.exists(tmpName): continue try: # takes care of only recent files modTime = datetime.datetime( *(time.gmtime(os.path.getmtime(tmpName))[:7])) if (timeNow - modTime) > datetime.timedelta(minutes=1) and \ (timeNow - modTime) < datetime.timedelta(hours=1): cSt, cOut = commands_get_status_output( 'ps aux | grep fork | grep -v PYTH') # if no process is running for the file if cSt == 0 and tmpName not in cOut: nThr += 1 thr = ForkThr(tmpName) thr.start() forkThrList.append(thr) if nThr > maxThr: break except Exception: errType, errValue = sys.exc_info()[:2] tmpLog.error("%s %s" % (errType, errValue)) # join fork threads for thr in forkThrList: thr.join() # terminate TaskBuffer IF # taskBufferIF.terminate() tmpLog.debug("===================== end =====================")
def main(argv=tuple(), tbuf=None, **kwargs): try: long except NameError: long = int prelock_pid = GenericThread().get_pid() tmpLog = LogWrapper(_logger, "<pid={}>".format(prelock_pid)) tmpLog.debug("===================== start =====================") # return value, true to run main again in next daemon loop ret_val = True # grace period try: gracePeriod = int(argv[1]) except Exception: gracePeriod = 1 # lock interval in minutes lock_interval = 10 # retry interval in minutes retry_interval = 3 # instantiate TB if tbuf is None: from pandaserver.taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) else: taskBuffer = tbuf # instantiate sitemapper aSiteMapper = SiteMapper(taskBuffer) # thread for adder class AdderThread(GenericThread): def __init__(self, taskBuffer, aSiteMapper, job_output_reports): GenericThread.__init__(self) self.taskBuffer = taskBuffer self.aSiteMapper = aSiteMapper self.job_output_reports = job_output_reports # main loop def run(self): # initialize taskBuffer = self.taskBuffer aSiteMapper = self.aSiteMapper # get file list timeNow = datetime.datetime.utcnow() timeInt = datetime.datetime.utcnow() # unique pid GenericThread.__init__(self) uniq_pid = self.get_pid() # log pid tmpLog.debug("pid={0} : run".format(uniq_pid)) # stats n_processed = 0 # loop while True: # get report one_jor = self.job_output_reports.pop() if not one_jor: break # lock panda_id, job_status, attempt_nr, time_stamp = one_jor got_lock = taskBuffer.lockJobOutputReport( panda_id=panda_id, attempt_nr=attempt_nr, pid=uniq_pid, time_limit=lock_interval) if not got_lock: continue # add try: modTime = time_stamp if (timeNow - modTime) > datetime.timedelta(hours=24): # last add tmpLog.debug( "pid={0} : last add job={1}.{2} st={3}".format( uniq_pid, panda_id, attempt_nr, job_status)) ignoreTmpError = False else: # usual add tmpLog.debug("pid={0} : add job={1}.{2} st={3}".format( uniq_pid, panda_id, attempt_nr, job_status)) ignoreTmpError = True # get adder adder_gen = AdderGen(taskBuffer, panda_id, job_status, attempt_nr, ignoreTmpError=ignoreTmpError, siteMapper=aSiteMapper, pid=uniq_pid, prelock_pid=uniq_pid, lock_offset=lock_interval - retry_interval) n_processed += 1 # execute adder_gen.run() del adder_gen except Exception as e: tmpLog.error("pid={} : failed to run with {} {}".format( uniq_pid, str(e), traceback.format_exc())) # stats tmpLog.debug("pid={} : processed {}".format(uniq_pid, n_processed)) # launcher, run with multiprocessing def proc_launch(self): # run self.process = multiprocessing.Process(target=self.run) self.process.start() # join of multiprocessing def proc_join(self): self.process.join() # TaskBuffer with more connections behind TaskBufferInterface tmpLog.debug("setup taskBufferIF") n_connections = 4 _tbuf = TaskBuffer() _tbuf.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=n_connections) taskBufferIF = TaskBufferInterface() taskBufferIF.launch(_tbuf) # add files tmpLog.debug("run Adder") interval = 10 nLoop = 10 for iLoop in range(10): tmpLog.debug('start iLoop={}/{}'.format(iLoop, nLoop)) start_time = datetime.datetime.utcnow() adderThrList = [] nThr = 10 n_jors_per_batch = 1000 jor_lists = WeightedLists(multiprocessing.Lock()) # get some job output reports jor_list_others = taskBuffer.listJobOutputReport( only_unlocked=True, time_limit=lock_interval, limit=n_jors_per_batch * nThr, grace_period=gracePeriod, anti_labels=['user']) jor_lists.add(3, jor_list_others) jor_list_user = taskBuffer.listJobOutputReport( only_unlocked=True, time_limit=lock_interval, limit=n_jors_per_batch * nThr, grace_period=gracePeriod, labels=['user']) jor_lists.add(7, jor_list_user) # adder consumer processes _n_thr_with_tbuf = 0 tbuf_list = [] tmpLog.debug("got {} job reports".format(len(jor_lists))) for i in range(nThr): if i < _n_thr_with_tbuf: tbuf = TaskBuffer() tbuf_list.append(tbuf) tbuf.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) thr = AdderThread(tbuf, aSiteMapper, jor_lists) else: thr = AdderThread(taskBufferIF.getInterface(), aSiteMapper, jor_lists) adderThrList.append(thr) # start all threads for thr in adderThrList: # thr.start() thr.proc_launch() time.sleep(0.25) # join all threads for thr in adderThrList: # thr.join() thr.proc_join() [tbuf.cleanup() for tbuf in tbuf_list] end_time = datetime.datetime.utcnow() sleep_time = interval - (end_time - start_time).seconds if sleep_time > 0 and iLoop + 1 < nLoop: sleep_time = random.randint(1, sleep_time) tmpLog.debug("sleep {} sec".format(sleep_time)) time.sleep(sleep_time) # stop TaskBuffer IF taskBufferIF.stop() tmpLog.debug("===================== end =====================") # return return ret_val
def core_exec(sandbox_url, log_token, dump_workflow, ops_file, user_name, test_mode): tmpLog = LogWrapper(_logger, log_token) is_OK = True is_fatal = False request_id = None if dump_workflow == 'True': dump_workflow = True else: dump_workflow = False if test_mode == 'True': test_mode = True else: test_mode = False try: with open(ops_file) as f: ops = json.load(f) try: os.remove(ops_file) except Exception: pass # go to temp dir cur_dir = os.getcwd() with tempfile.TemporaryDirectory() as tmp_dirname: os.chdir(tmp_dirname) # download sandbox tmpLog.info('downloading sandbox from {}'.format(sandbox_url)) with requests.get(sandbox_url, allow_redirects=True, verify=False, stream=True) as r: if r.status_code == 400: tmpLog.error("not found") is_fatal = True is_OK = False elif r.status_code != 200: tmpLog.error("bad HTTP response {}".format(r.status_code)) is_OK = False # extract sandbox if is_OK: with open(ops['data']['sandbox'], 'wb') as fs: for chunk in r.raw.stream(1024, decode_content=False): if chunk: fs.write(chunk) fs.close() tmp_stat, tmp_out = commands_get_status_output( 'tar xvfz {}'.format(ops['data']['sandbox'])) if tmp_stat != 0: tmpLog.error(tmp_out) dump_str = 'failed to extract {}'.format(ops['data']['sandbox']) tmpLog.error(dump_str) is_fatal = True is_OK = False # parse workflow files if is_OK: tmpLog.info('parse workflow') if ops['data']['language'] == 'cwl': nodes, root_in = pcwl_utils.parse_workflow_file(ops['data']['workflowSpecFile'], tmpLog) with open(ops['data']['workflowInputFile']) as workflow_input: data = yaml.safe_load(workflow_input) s_id, t_nodes, nodes = pcwl_utils.resolve_nodes(nodes, root_in, data, 0, set(), ops['data']['outDS'], tmpLog) workflow_utils.set_workflow_outputs(nodes) id_node_map = workflow_utils.get_node_id_map(nodes) [node.resolve_params(ops['data']['taskParams'], id_node_map) for node in nodes] dump_str = "the description was internally converted as follows\n" \ + workflow_utils.dump_nodes(nodes) tmpLog.info(dump_str) for node in nodes: s_check, o_check = node.verify() tmp_str = 'Verification failure in ID:{} {}'.format(node.id, o_check) if not s_check: tmpLog.error(tmp_str) dump_str += tmp_str dump_str += '\n' is_fatal = True is_OK = False else: dump_str = "{} is not supported to describe the workflow" tmpLog.error(dump_str) is_fatal = True is_OK = False # convert to workflow if is_OK: workflow_to_submit, dump_str_list = workflow_utils.convert_nodes_to_workflow(nodes) try: if workflow_to_submit: if not test_mode: tmpLog.info('submit workflow') wm = ClientManager(host=get_rest_host()) request_id = wm.submit(workflow_to_submit, username=user_name) else: dump_str = 'workflow is empty' tmpLog.error(dump_str) is_fatal = True is_OK = False except Exception as e: dump_str = 'failed to submit the workflow with {}'.format(str(e)) tmpLog.error('{} {}'.format(dump_str, traceback.format_exc())) if dump_workflow: tmpLog.debug('\n' + ''.join(dump_str_list)) os.chdir(cur_dir) except Exception as e: is_OK = False is_fatal = True tmpLog.error("failed to run with {} {}".format(str(e), traceback.format_exc())) with tempfile.NamedTemporaryFile(delete=False, mode='w') as tmp_json: json.dump([is_OK, is_fatal, request_id, tmpLog.dumpToString()], tmp_json) print(tmp_json.name) sys.exit(0)
def process(self, file_name, to_delete=False, test_mode=False, get_log=False, dump_workflow=False): try: is_fatal = False is_OK = True request_id = None dump_str = None with open(file_name) as f: ops = json.load(f) user_name = clean_user_id(ops["userName"]) base_platform = ops['data'].get('base_platform') for task_type in ops['data']['taskParams']: ops['data']['taskParams'][task_type]['userName'] = user_name if base_platform: ops['data']['taskParams'][task_type]['basePlatform'] = base_platform log_token = '< id="{}" test={} outDS={} >'.format(user_name, test_mode, ops['data']['outDS']) tmpLog = LogWrapper(self.log, log_token) tmpLog.info('start {}'.format(file_name)) sandbox_url = os.path.join(ops['data']['sourceURL'], 'cache', ops['data']['sandbox']) # IO through json files ops_file = tempfile.NamedTemporaryFile(delete=False, mode='w') json.dump(ops, ops_file) ops_file.close() # execute main in another process to avoid chdir mess tmp_stat, tmp_out = commands_get_status_output("python {} {} '{}' {} {} '{}' {}".format( __file__, sandbox_url, log_token, dump_workflow, ops_file.name, user_name, test_mode)) if tmp_stat: is_OK = False tmpLog.error('main execution failed with {}:{}'.format(tmp_stat, tmp_out)) else: with open(tmp_out.split('\n')[-1]) as tmp_out_file: is_OK, is_fatal, request_id, dump_str = json.load(tmp_out_file) try: os.remove(tmp_out) except Exception: pass if not get_log: if is_OK: tmpLog.info('is_OK={} request_id={}'.format(is_OK, request_id)) else: tmpLog.info('is_OK={} is_fatal={} request_id={}'.format(is_OK, is_fatal, request_id)) if to_delete or (not test_mode and (is_OK or is_fatal)): dump_str = tmpLog.dumpToString() + dump_str tmpLog.debug('delete {}'.format(file_name)) try: os.remove(file_name) except Exception: pass # send notification if not test_mode and self.taskBuffer is not None: toAdder = self.taskBuffer.getEmailAddr(user_name) if toAdder is None or toAdder.startswith('notsend'): tmpLog.debug('skip to send notification since suppressed') else: # message if is_OK: mailSubject = "PANDA Notification for Workflow {}".format(ops['data']['outDS']) mailBody = "Hello,\n\nWorkflow:{} has been accepted with RequestID:{}\n\n".\ format(ops['data']['outDS'], request_id) else: mailSubject = "PANDA WARNING for Workflow={}".format(ops['data']['outDS']) mailBody = "Hello,\n\nWorkflow {} was not accepted\n\n".\ format(ops['data']['outDS'], request_id) mailBody += "Reason : %s\n" % dump_str # send tmpSM = MailUtils().send(toAdder, mailSubject, mailBody) tmpLog.debug('sent message with {}'.format(tmpSM)) except Exception as e: is_OK = False tmpLog.error("failed to run with {} {}".format(str(e), traceback.format_exc())) if get_log: ret_val = {'status': is_OK} if is_OK: ret_val['log'] = dump_str else: if dump_str is None: ret_val['log'] = tmpLog.dumpToString() else: ret_val['log'] = dump_str return ret_val
def application(environ, start_response): # get method name methodName = '' if 'SCRIPT_NAME' in environ: methodName = environ['SCRIPT_NAME'].split('/')[-1] tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName)) tmpLog.debug("start") regStart = datetime.datetime.utcnow() retType = None # check method name if not methodName in allowedMethods: tmpLog.error("is forbidden") exeRes = "False : %s is forbidden" % methodName else: # get method object tmpMethod = None try: tmpMethod = globals()[methodName] except Exception: pass # object not found if tmpMethod is None: tmpLog.error("is undefined") exeRes = "False" else: try: # get params tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ, keep_blank_values=1) # convert to map params = {} for tmpKey in list(tmpPars): if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None: # file params[tmpKey] = tmpPars[tmpKey] else: # string params[tmpKey] = tmpPars.getfirst(tmpKey) if panda_config.entryVerbose: tmpLog.debug("with %s" % str(list(params))) # dummy request object dummyReq = DummyReq(environ, tmpLog) param_list = [dummyReq] # exec exeRes = tmpMethod(*param_list, **params) # extract return type if isinstance(exeRes, dict): retType = exeRes['type'] exeRes = exeRes['content'] # convert bool to string if exeRes in [True,False]: exeRes = str(exeRes) except Exception as e: tmpLog.error("execution failure : {0}".format(str(e))) errStr = "" for tmpKey in environ: tmpVal = environ[tmpKey] errStr += "%s : %s\n" % (tmpKey,str(tmpVal)) tmpLog.error(errStr) # return internal server error start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')]) return [str(e)] if panda_config.entryVerbose: tmpLog.debug("done") regTime = datetime.datetime.utcnow() - regStart tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds, regTime.microseconds/1000, len(str(exeRes)))) # return if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound: start_response('404 Not Found', [('Content-Type', 'text/plain')]) return ['not found'] elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect): start_response('302 Redirect', [('Location', exeRes.url)]) return ['redirect'] else: if retType == 'json': start_response('200 OK', [('Content-Type', 'application/json')]) else: start_response('200 OK', [('Content-Type', 'text/plain')]) if isinstance(exeRes, str): exeRes = exeRes.encode() return [exeRes]
def __init__(self, job, datasets, log): self.jobSpec = job self.datasets = datasets self.tmpLog = LogWrapper( log, "{0} CloserAtlasPlugin".format(self.jobSpec.PandaID))
def run(self): try: # make a message instance tmpLog = LogWrapper(_logger, None) # run main procedure in the same process if not self.forkRun: tmpLog.debug('main start') tmpLog.debug('firstSubmission={0}'.format( self.firstSubmission)) # group jobs per VO voJobsMap = {} ddmFreeJobs = [] tmpLog.debug('{0} jobs in total'.format(len(self.jobs))) for tmpJob in self.jobs: # set VO=local for DDM free if tmpJob.destinationSE == 'local': tmpVO = 'local' else: tmpVO = tmpJob.VO # make map voJobsMap.setdefault(tmpVO, []) voJobsMap[tmpVO].append(tmpJob) # loop over all VOs for tmpVO in voJobsMap: tmpJobList = voJobsMap[tmpVO] tmpLog.debug('vo={0} has {1} jobs'.format( tmpVO, len(tmpJobList))) # get plugin setupperPluginClass = panda_config.getPlugin( 'setupper_plugins', tmpVO) if setupperPluginClass is None: # use ATLAS plug-in by default from pandaserver.dataservice.SetupperAtlasPlugin import SetupperAtlasPlugin setupperPluginClass = SetupperAtlasPlugin tmpLog.debug('plugin name -> {0}'.format( setupperPluginClass.__name__)) try: # make plugin setupperPlugin = setupperPluginClass( self.taskBuffer, self.jobs, tmpLog, resubmit=self.resubmit, pandaDDM=self.pandaDDM, ddmAttempt=self.ddmAttempt, onlyTA=self.onlyTA, firstSubmission=self.firstSubmission) # run plugin tmpLog.debug('run plugin') setupperPlugin.run() # go forward if not TA if not self.onlyTA: # update jobs tmpLog.debug('update jobs') self.updateJobs( setupperPlugin.jobs + setupperPlugin.jumboJobs, tmpLog) # execute post process tmpLog.debug('post execute plugin') setupperPlugin.postRun() tmpLog.debug('done plugin') except Exception: errtype, errvalue = sys.exc_info()[:2] tmpLog.error('plugin failed with {0}:{1}'.format( errtype, errvalue)) tmpLog.debug('main end') else: tmpLog.debug('fork start') # write jobs to file import os try: import cPickle as pickle except ImportError: import pickle outFileName = '%s/set.%s_%s' % (panda_config.logdir, self.jobs[0].PandaID, str(uuid.uuid4())) outFile = open(outFileName, 'wb') pickle.dump(self.jobs, outFile, protocol=0) outFile.close() # run main procedure in another process because python doesn't release memory com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % ( panda_config.home_dir_cwd, panda_config.home_dir_cwd) com += 'env PYTHONPATH=%s:%s %s/python -Wignore %s/dataservice/forkSetupper.py -i %s' % \ (panda_config.pandaCommon_dir,panda_config.pandaPython_dir,panda_config.native_python, panda_config.pandaPython_dir,outFileName) if self.onlyTA: com += " -t" if not self.firstSubmission: com += " -f" tmpLog.debug(com) # execute status, output = self.taskBuffer.processLimiter.getstatusoutput( com) tmpLog.debug("return from main process: %s %s" % (status, output)) tmpLog.debug('fork end') except Exception as e: tmpLog.error('master failed with {0} {1}'.format( str(e), traceback.format_exc()))
import re import sys import datetime import traceback from pandaserver.taskbuffer.TaskBuffer import taskBuffer from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.brokerage.SiteMapper import SiteMapper # password from pandaserver.config import panda_config passwd = panda_config.dbpasswd # logger _logger = PandaLogger().getLogger('prioryMassage') tmpLog = LogWrapper(_logger) tmpLog.debug("================= start ==================") # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # get usage breakdown usageBreakDownPerUser = {} usageBreakDownPerSite = {} workingGroupList = [] for table in ['ATLAS_PANDA.jobsActive4', 'ATLAS_PANDA.jobsArchived4']: varMap = {}
def getGUIDsFromEventIndex(self, runEventList, streamName, amiTags, dataType): comment = ' /* DBProxy.getGUIDsFromEventIndex */' methodName = comment.split(' ')[-2].split('.')[-1] tmpLog = LogWrapper( _logger, methodName + " <streamName={0} amiTags={1} dataType={2}>".format( streamName, amiTags, dataType)) try: # change to list if not amiTags in [None, '']: amiTags = amiTags.replace('*', '.*').split(',') tmpLog.debug("start for {0} events".format(len(runEventList))) # check data type if not dataType in ['RAW', 'ESD', 'AOD']: return False, 'dataType={0} is unsupported'.format(dataType) # sql to insert runs and events sqlRE = "INSERT INTO {0}.TMP_RUN_EVENT_PAIRS (runNumber,eventNumber) ".format( panda_config.schemaEI) sqlRE += "VALUES (:runNumber,:eventNumber) " varMaps = [] for runNumber, eventNumber in runEventList: varMap = {} varMap[':runNumber'] = runNumber varMap[':eventNumber'] = eventNumber varMaps.append(varMap) # begin transaction self.conn.begin() self.cur.arraysize = 100000 # insert runs and events self.cur.executemany(sqlRE + comment, varMaps) # read GUIDs varMap = {} if amiTags in [None, '']: sqlRG = "SELECT runNumber,eventNumber,guid_{0} ".format( dataType) sqlRG += "FROM {0}.V_PANDA_EVPICK_NOAMITAG_MANY ".format( panda_config.schemaEI) else: sqlRG = "SELECT runNumber,eventNumber,guid_{0},amiTag ".format( dataType) sqlRG += "FROM {0}.V_PANDA_EVPICK_AMITAG_MANY ".format( panda_config.schemaEI) if not streamName in [None, '']: sqlRG += "WHERE streamName=:streamName " varMap[':streamName'] = streamName self.cur.execute(sqlRG + comment, varMap) resRG = self.cur.fetchall() # commit if not self._commit(): raise RuntimeError('Commit error') retValue = {} keyAmiIdxMap = {} for tmpItem in resRG: if amiTags in [None, '']: runNumber, eventNumber, guid = tmpItem # dummy idxTag = 0 else: runNumber, eventNumber, guid, amiTag = tmpItem # get index number for the AMI tag in the list idxTag = self.getIndexAmiTag(amiTags, amiTag) # didn't match if idxTag is None: continue tmpKey = (runNumber, eventNumber) # use AMI tag in a preference orde if tmpKey in keyAmiIdxMap and keyAmiIdxMap[tmpKey] < idxTag: continue keyAmiIdxMap[tmpKey] = idxTag retValue[tmpKey] = [guid] tmpLog.debug("found {0} events".format(len(retValue))) return True, retValue except Exception: # roll back self._rollback() # error self.dumpErrorMessage(_logger, methodName) return False, None
from pandaserver.srvcore.CoreUtils import commands_get_status_output from pandaserver.taskbuffer.TaskBufferInterface import TaskBufferInterface try: long except NameError: long = int # password from pandaserver.config import panda_config # logger _logger = PandaLogger().getLogger('add') tmpLog = LogWrapper(_logger, None) tmpLog.debug("===================== start =====================") # overall timeout value overallTimeout = 20 # grace period try: gracePeriod = int(sys.argv[1]) except Exception: gracePeriod = 3 # current minute currentMinute = datetime.datetime.utcnow().minute
def application(environ, start_response): # get method name methodName = '' if 'SCRIPT_NAME' in environ: methodName = environ['SCRIPT_NAME'].split('/')[-1] tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName), seeMem=True) cont_length = int(environ.get('CONTENT_LENGTH', 0)) json_body = environ.get('CONTENT_TYPE', None) == 'application/json' tmpLog.debug("start content-length={} json={}".format(cont_length, json_body)) regStart = datetime.datetime.utcnow() retType = None # check method name if methodName not in allowedMethods: tmpLog.error("is forbidden") exeRes = "False : %s is forbidden" % methodName else: # get method object tmpMethod = None try: tmpMethod = globals()[methodName] except Exception: pass # object not found if tmpMethod is None: tmpLog.error("is undefined") exeRes = "False" else: body = b'' try: # dummy request object dummyReq = DummyReq(environ, tmpLog) if not dummyReq.authenticated: start_response('403 Forbidden', [('Content-Type', 'text/plain')]) return ["ERROR : Token authentication failed on the server side. {}".format( dummyReq.message).encode()] username = dummyReq.subprocess_env.get('SSL_CLIENT_S_DN', None) if username: username = CoreUtils.clean_user_id(username) if username in ban_user_list: errMsg = '{} is banned'.format(username) tmpLog.warning(errMsg) start_response('403 Forbidden', [('Content-Type', 'text/plain')]) return ["ERROR : {}".format(errMsg).encode()] # read contents while cont_length > 0: chunk = environ['wsgi.input'].read(min(cont_length, 1024*1024)) if not chunk: break cont_length -= len(chunk) body += chunk if cont_length > 0: raise OSError('partial read from client. {} bytes remaining'.format(cont_length)) if not json_body: # query string environ['wsgi.input'] = io.BytesIO(body) # get params tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ, keep_blank_values=1) # convert to map params = {} for tmpKey in list(tmpPars): if tmpPars[tmpKey].file is not None and tmpPars[tmpKey].filename is not None: # file params[tmpKey] = tmpPars[tmpKey] else: # string params[tmpKey] = tmpPars.getfirst(tmpKey) else: # json body = gzip.decompress(body) params = json.loads(body) if panda_config.entryVerbose: tmpLog.debug("with %s" % str(list(params))) param_list = [dummyReq] # exec exeRes = tmpMethod(*param_list, **params) # extract return type if isinstance(exeRes, dict): retType = exeRes['type'] exeRes = exeRes['content'] # convert bool to string if exeRes in [True,False]: exeRes = str(exeRes) except Exception as e: tmpLog.error("execution failure : {0}\n {1}".format(str(e), traceback.format_exc())) if hasattr(panda_config, 'dumpBadRequest') and panda_config.dumpBadRequest: try: with tempfile.NamedTemporaryFile(delete=False, prefix='req_dump_') as f: environ['WSGI_INPUT_DUMP'] = f.name f.write(body) os.chmod(f.name, 0o775) except Exception: tmpLog.error(traceback.format_exc()) pass errStr = "" for tmpKey in environ: tmpVal = environ[tmpKey] errStr += "%s : %s\n" % (tmpKey,str(tmpVal)) tmpLog.error(errStr) # return internal server error start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')]) # force kill to release memory if type(e) == OSError: tmpLog.warning('force restart due') os.kill(os.getpid(), signal.SIGINT) return [str(e).encode()] if panda_config.entryVerbose: tmpLog.debug("done") regTime = datetime.datetime.utcnow() - regStart tmpLog.info("exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds, regTime.microseconds/1000, len(str(exeRes)))) # return if exeRes == pandaserver.taskbuffer.ErrorCode.EC_NotFound: start_response('404 Not Found', [('Content-Type', 'text/plain')]) return ['not found'.encode()] elif isinstance(exeRes, pandaserver.taskbuffer.ErrorCode.EC_Redirect): start_response('302 Redirect', [('Location', exeRes.url)]) return ['redirect'.encode()] else: if retType == 'json': start_response('200 OK', [('Content-Type', 'application/json')]) else: start_response('200 OK', [('Content-Type', 'text/plain')]) if isinstance(exeRes, str): exeRes = exeRes.encode() return [exeRes]