def put_file_recovery_request(req, jediTaskID, dryRun=None): if not Protocol.isSecure(req): return json.dumps((False, "ERROR : no HTTPS")) userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') tmpLog = LogWrapper(_logger, 'put_file_recovery_request < jediTaskID={}'.format(jediTaskID)) tmpLog.debug("start user={}".format(userName)) # get total size try: jediTaskID = int(jediTaskID) # make filename evpFileName = '%s/recov.%s' % (panda_config.cache_dir,str(uuid.uuid4())) tmpLog.debug("file={}".format(evpFileName)) # write with open(evpFileName, 'w') as fo: data = {"userName": userName, "creationTime": creationTime, "jediTaskID": int(jediTaskID) } if dryRun: data['dryRun'] = True json.dump(data, fo) except Exception as e: errStr = "cannot put request due to {} ".format(str(e)) tmpLog.error(errStr + traceback.format_exc()) return json.dumps((False, errStr)) tmpLog.debug('done') return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def deleteFile(req, file): if not Protocol.isSecure(req): return 'False' try: # may be reused for rebrokreage #os.remove('%s/%s' % (panda_config.cache_dir,file.split('/')[-1])) return 'True' except Exception: return 'False'
def touchFile(req,filename): if not Protocol.isSecure(req): return 'False' try: os.utime('%s/%s' % (panda_config.cache_dir,filename.split('/')[-1]),None) return 'True' except Exception: errtype,errvalue = sys.exc_info()[:2] _logger.error("touchFile : %s %s" % (errtype,errvalue)) return 'False'
def uploadLog(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename)) tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN'])) # size check sizeLimit = 100 * 1024 * 1024 # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except Exception: if "content-length" in req.headers_in: tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "failed to upload log due to size limit" tmpLog.error(errStr) tmpLog.debug("end") return errStr jediLogDir = '/jedilog' retStr = '' try: fileBaseName = file.filename.split('/')[-1] fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir, fileBaseName) # delete old file if os.path.exists(fileFullPath): os.remove(fileFullPath) # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() tmpLog.debug("written to {0}".format(fileFullPath)) retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None), jediLogDir, fileBaseName) except Exception: errtype, errvalue = sys.exc_info()[:2] errStr = "failed to write log with {0}:{1}".format( errtype.__name__, errvalue) tmpLog.error(errStr) tmpLog.debug("end") return errStr tmpLog.debug("end") return retStr
def delete_checkpoint(req, task_id, sub_id): tmpLog = LogWrapper(_logger, 'delete_checkpoint <jediTaskID={0} ID={1}>'.format(task_id, sub_id)) status = True if not Protocol.isSecure(req): msg = 'insecure request' tmpLog.error(msg) status = False else: tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN']) try: fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id)) os.remove(fileFullPath) msg = 'done' tmpLog.debug(msg) except Exception as e: msg = "failed to delete file due to {0}".format(str(e)) tmpLog.error(msg) status = False return json.dumps({'status': status, 'message': msg})
def put_checkpoint(req, file): tmpLog = LogWrapper(_logger, 'put_checkpoint <jediTaskID_subID={0}>'.format(file.filename)) status = False if not Protocol.isSecure(req): errStr = 'insecure request' tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN']) # extract taskID and subID try: task_id, sub_id = file.filename.split('/')[-1].split('_') except Exception: errStr = 'failed to extract ID' tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) # size check sizeLimit = 500 * 1024 * 1024 # get file size try: contentLength = long(req.headers_in["content-length"]) except Exception as e: errStr = "cannot get int(content-length) due to {0}".format(str(e)) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "exceeded size limit %s>%s" % (contentLength, sizeLimit) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) try: fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id)) # write with open(fileFullPath,'wb') as fo: fo.write(file.file.read()) except Exception as e: errStr = "cannot write file due to {0}".format(str(e)) tmpLog.error(errStr) return json.dumps({'status': status, 'message': errStr}) status = True tmpMsg = "successfully placed at {0}".format(fileFullPath) tmpLog.debug(tmpMsg) return json.dumps({'status': status, 'message': tmpMsg})
def put_workflow_request(req, data, check=False): if not Protocol.isSecure(req): return json.dumps((False, "ERROR : no HTTPS")) userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') tmpLog = LogWrapper(_logger, 'put_workflow_request') tmpLog.debug("start user={} check={}".format(userName, check)) if check == 'True' or check is True: check = True else: check = False # get total size try: # make filename evpFileName = '%s/workflow.%s' % (panda_config.cache_dir,str(uuid.uuid4())) tmpLog.debug("file={}".format(evpFileName)) # write with open(evpFileName, 'w') as fo: data = {"userName": userName, "creationTime": creationTime, "data": json.loads(data), } json.dump(data, fo) # check if check: tmpLog.debug('checking') from pandaserver.taskbuffer.workflow_processor import WorkflowProcessor processor = WorkflowProcessor(log_stream=_logger) ret = processor.process(evpFileName, True, True, True, True) if os.path.exists(evpFileName): try: os.remove(evpFileName) except Exception: pass tmpLog.debug('done') return json.dumps((True, ret)) except Exception as e: errStr = "cannot put request due to {} ".format(str(e)) tmpLog.error(errStr + traceback.format_exc()) return json.dumps((False, errStr)) tmpLog.debug('done') return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def putEventPickingRequest(req, runEventList='', eventPickDataType='', eventPickStreamName='', eventPickDS='', eventPickAmiTag='', userDatasetName='', lockedBy='', params='', inputFileList='', eventPickNumSites='', userTaskName='', ei_api='', giveGUID=None): if not Protocol.isSecure(req): return "ERROR : no HTTPS" userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') _logger.debug("putEventPickingRequest : %s start" % userName) # size check sizeLimit = 10 * 1024 * 1024 # get total size try: contentLength = long(req.headers_in["content-length"]) except Exception: errStr = "cannot get content-length from HTTP request." _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "Too large run/event list. Exceeded size limit %s>%s." % ( contentLength, sizeLimit) _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr if giveGUID == 'True': giveGUID = True else: giveGUID = False try: # make filename evpFileName = '%s/evp.%s' % (panda_config.cache_dir, str(uuid.uuid4())) _logger.debug("putEventPickingRequest : %s -> %s" % (userName, evpFileName)) # write fo = open(evpFileName, 'wb') fo.write("userName=%s\n" % userName) fo.write("creationTime=%s\n" % creationTime) fo.write("eventPickDataType=%s\n" % eventPickDataType) fo.write("eventPickStreamName=%s\n" % eventPickStreamName) fo.write("eventPickDS=%s\n" % eventPickDS) fo.write("eventPickAmiTag=%s\n" % eventPickAmiTag) fo.write("eventPickNumSites=%s\n" % eventPickNumSites) fo.write("userTaskName=%s\n" % userTaskName) fo.write("userDatasetName=%s\n" % userDatasetName) fo.write("lockedBy=%s\n" % lockedBy) fo.write("params=%s\n" % params) fo.write("inputFileList=%s\n" % inputFileList) fo.write("ei_api=%s\n" % ei_api) runEvtGuidMap = {} for tmpLine in runEventList.split('\n'): tmpItems = tmpLine.split() if (len(tmpItems) != 2 and not giveGUID) or \ (len(tmpItems) != 3 and giveGUID): continue fo.write("runEvent=%s,%s\n" % tuple(tmpItems[:2])) if giveGUID: runEvtGuidMap[tuple(tmpItems[:2])] = [tmpItems[2]] fo.write("runEvtGuidMap=%s\n" % str(runEvtGuidMap)) fo.close() except Exception: errType, errValue = sys.exc_info()[:2] errStr = "cannot put request due to %s %s" % (errType, errValue) _logger.error("putEventPickingRequest : " + errStr + " " + userName) return "ERROR : " + errStr _logger.debug("putEventPickingRequest : %s end" % userName) return True
def putFile(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False _logger.debug("putFile : start %s %s" % (req.subprocess_env['SSL_CLIENT_S_DN'], file.filename)) # size check fullSizeLimit = 768 * 1024 * 1024 if not file.filename.startswith('sources.'): noBuild = True sizeLimit = 100 * 1024 * 1024 else: noBuild = False sizeLimit = fullSizeLimit # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except Exception: if "content-length" in req.headers_in: _logger.error("cannot get CL : %s" % req.headers_in["content-length"]) else: _logger.error("no CL") _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % ( contentLength, sizeLimit) if noBuild: errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit" else: errStr += " Please remove redundant files from your workarea" _logger.error(errStr) _logger.debug("putFile : end") return errStr try: fileFullPath = '%s/%s' % (panda_config.cache_dir, file.filename.split('/')[-1]) # avoid overwriting if os.path.exists(fileFullPath): # touch os.utime(fileFullPath, None) # send error message errStr = "ERROR : Cannot overwrite file" _logger.debug('putFile : cannot overwrite file %s' % file.filename) _logger.debug("putFile : end") return errStr # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() except Exception: errStr = "ERROR : Cannot write file" _logger.error(errStr) _logger.debug("putFile : end") return errStr # checksum try: # decode Footer footer = fileContent[-8:] checkSum, isize = struct.unpack("II", footer) _logger.debug("CRC from gzip Footer %s" % checkSum) except Exception: # calculate on the fly """ import zlib checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF """ # use None to avoid delay for now checkSum = None _logger.debug("CRC calculated %s" % checkSum) # file size fileSize = len(fileContent) # user name username = cleanUserID(req.subprocess_env['SSL_CLIENT_S_DN']) _logger.debug("putFile : written dn=%s file=%s size=%s crc=%s" % \ (username,file.filename,fileSize,checkSum)) # put file info to DB statClient, outClient = Client.insertSandboxFileInfo( username, file.filename, fileSize, checkSum) if statClient != 0 or outClient.startswith("ERROR"): _logger.error("putFile : failed to put sandbox to DB with %s %s" % (statClient, outClient)) #_logger.debug("putFile : end") #return "ERROR : Cannot insert sandbox to DB" else: _logger.debug("putFile : inserted sandbox to DB with %s" % outClient) # store to cassandra if hasattr(panda_config, 'cacheUseCassandra') and panda_config.cacheUseCassandra == True: try: # time-stamp timeNow = datetime.datetime.utcnow() creationTime = timeNow.strftime('%Y-%m-%d %H:%M:%S') # user name username = req.subprocess_env['SSL_CLIENT_S_DN'] username = username.replace('/CN=proxy', '') username = username.replace('/CN=limited proxy', '') # file size fileSize = len(fileContent) # key fileKeyName = file.filename.split('/')[-1] sizeCheckSum = '%s:%s' % (fileSize, checkSum) # insert to cassandra import pycassa pool = pycassa.ConnectionPool(panda_config.cacheKeySpace) filefamily = pycassa.ColumnFamily(pool, panda_config.cacheFileTable) # avoid overwriting gotoNextCassa = True if filefamily.get_count(fileKeyName) > 0: # touch touchFlag = touchFileCassa(filefamily, fileKeyName, timeNow) if touchFlag: gotoNextCassa = False # send error message errStr = "ERROR : Cannot overwrite file in Cassandra" _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # check uniqueness with size and checksum if gotoNextCassa: try: uniqExp = pycassa.index.create_index_expression( 'uniqID', sizeCheckSum) userExp = pycassa.index.create_index_expression( 'user', username) tmpClause = pycassa.index.create_index_clause( [uniqExp, userExp]) tmpResults = filefamily.get_indexed_slices( tmpClause, columns=['creationTime']) for oldFileKeyName, tmpDict in tmpResults: _logger.debug('The same size and chksum %s found in old:%s and new:%s' % \ (sizeCheckSum,oldFileKeyName,fileKeyName)) # touch touchFlag = touchFileCassa(filefamily, oldFileKeyName, timeNow) if touchFlag: # make alias _logger.debug('Making alias %s->%s' % (fileKeyName, oldFileKeyName)) insertWithRetryCassa( filefamily, fileKeyName, { 'alias': oldFileKeyName, 'creationTime': creationTime, 'nSplit': 0, }, 'putFile : make alias for %s' % file.filename) # set time touchFileCassa(filefamily, fileKeyName, timeNow) _logger.debug("putFile : end") return True except Exception: gotoNextCassa = False errType, errValue = sys.exc_info()[:2] errStr = "cannot make alias for %s due to %s %s" % ( fileKeyName, errType, errValue) _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # insert new record if gotoNextCassa: splitIdx = 0 splitSize = 5 * 1024 * 1024 nSplit, tmpMod = divmod(len(fileContent), splitSize) if tmpMod != 0: nSplit += 1 _logger.debug('Inserting %s with %s blocks' % (fileKeyName, nSplit)) for splitIdx in range(nSplit): # split to small chunks since cassandra is not good at large files tmpFileContent = fileContent[splitSize * splitIdx:splitSize * (splitIdx + 1)] tmpFileKeyName = fileKeyName tmpAttMap = { 'file': tmpFileContent, 'user': username, 'creationTime': creationTime, } if splitIdx == 0: tmpAttMap['size'] = fileSize tmpAttMap['nSplit'] = nSplit tmpAttMap['uniqID'] = sizeCheckSum tmpAttMap['checkSum'] = str(checkSum) else: tmpFileKeyName += '_%s' % splitIdx tmpAttMap['size'] = 0 tmpAttMap['nSplit'] = 0 # insert with retry insertWithRetryCassa(filefamily, tmpFileKeyName, tmpAttMap, 'putFile : insert %s' % file.filename) # set time touchFileCassa(filefamily, fileKeyName, timeNow) except Exception: errType, errValue = sys.exc_info()[:2] errStr = "cannot put %s into Cassandra due to %s %s" % ( fileKeyName, errType, errValue) _logger.error(errStr) # send error message errStr = "ERROR : " + errStr if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr _logger.debug("putFile : %s end" % file.filename) return True
def putFile(req, file): tmpLog = LogWrapper(_logger, 'putFile-{}'.format(datetime.datetime.utcnow().isoformat('/'))) if not Protocol.isSecure(req): tmpLog.error('No SSL_CLIENT_S_DN') return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False # user name username = CoreUtils.clean_user_id(req.subprocess_env['SSL_CLIENT_S_DN']) tmpLog.debug("start %s %s" % (username, file.filename)) # size check fullSizeLimit = 768*1024*1024 if not file.filename.startswith('sources.'): noBuild = True sizeLimit = 100*1024*1024 else: noBuild = False sizeLimit = fullSizeLimit # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except Exception: if "content-length" in req.headers_in: tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % (contentLength,sizeLimit) if noBuild: errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit" else: errStr += " Please remove redundant files from your workarea" tmpLog.error(errStr) tmpLog.debug("end") return errStr try: fileName = file.filename.split('/')[-1] fileFullPath = '%s/%s' % (panda_config.cache_dir, fileName) # avoid overwriting if os.path.exists(fileFullPath): # touch os.utime(fileFullPath,None) # send error message errStr = "ERROR : Cannot overwrite file" tmpLog.debug('cannot overwrite file %s' % fileName) tmpLog.debug("end") return errStr # write fo = open(fileFullPath,'wb') fileContent = file.file.read() if hasattr(panda_config, 'compress_file_names') and \ [True for patt in panda_config.compress_file_names.split(',') if re.search(patt, fileName) is not None]: fileContent = gzip.compress(fileContent) fo.write(fileContent) fo.close() except Exception: errStr = "ERROR : Cannot write file" tmpLog.error(errStr) tmpLog.debug("end") return errStr # checksum try: # decode Footer footer = fileContent[-8:] checkSum,isize = struct.unpack("II",footer) tmpLog.debug("CRC from gzip Footer %s" % checkSum) except Exception: # calculate on the fly """ import zlib checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF """ # use None to avoid delay for now checkSum = None tmpLog.debug("CRC calculated %s" % checkSum) # file size fileSize = len(fileContent) tmpLog.debug("written dn=%s file=%s size=%s crc=%s" % \ (username, fileFullPath, fileSize, checkSum)) # put file info to DB if panda_config.record_sandbox_info: to_insert = True for patt in IGNORED_SUFFIX: if file.filename.endswith(patt): to_insert = False break if not to_insert: tmpLog.debug("skipped to insert to DB") else: statClient,outClient = Client.insertSandboxFileInfo(username,file.filename, fileSize,checkSum) if statClient != 0 or outClient.startswith("ERROR"): tmpLog.error("failed to put sandbox to DB with %s %s" % (statClient,outClient)) #_logger.debug("putFile : end") #return "ERROR : Cannot insert sandbox to DB" else: tmpLog.debug("inserted sandbox to DB with %s" % outClient) tmpLog.debug("end") return True