def deleteFile(req,file): if not Protocol.isSecure(req): return 'False' try: # may be reused for rebrokreage #os.remove('%s/%s' % (panda_config.cache_dir,file.split('/')[-1])) return 'True' except: return 'False'
def deleteFile(req, file): if not Protocol.isSecure(req): return 'False' try: # may be reused for rebrokreage #os.remove('%s/%s' % (panda_config.cache_dir,file.split('/')[-1])) return 'True' except: return 'False'
def isSecure(req): # check security if not Protocol.isSecure(req): return False # disable limited proxy if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: _logger.warning("access via limited proxy : %s" % req.subprocess_env['SSL_CLIENT_S_DN']) return False return True
def touchFile(req,filename): if not Protocol.isSecure(req): return 'False' try: os.utime('%s/%s' % (panda_config.cache_dir,filename.split('/')[-1]),None) return 'True' except: errtype,errvalue = sys.exc_info()[:2] _logger.error("touchFile : %s %s" % (errtype,errvalue)) return 'False'
def putEventPickingRequest(req,runEventList='',eventPickDataType='',eventPickStreamName='', eventPickDS='',eventPickAmiTag='',userDatasetName='',lockedBy='', params='',inputFileList='',eventPickNumSites='',userTaskName=''): if not Protocol.isSecure(req): return "ERROR : no HTTPS" userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') _logger.debug("putEventPickingRequest : %s start" % userName) # size check sizeLimit = 10*1024*1024 # get total size try: contentLength = long(req.headers_in["content-length"]) except: errStr = "cannot get content-length from HTTP request." _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "Too large run/event list. Exceeded size limit %s>%s." % (contentLength,sizeLimit) _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr try: # make filename evpFileName = '%s/evp.%s' % (panda_config.cache_dir,str(uuid.uuid4())) _logger.debug("putEventPickingRequest : %s -> %s" % (userName,evpFileName)) # write fo = open(evpFileName,'wb') fo.write("userName=%s\n" % userName) fo.write("creationTime=%s\n" % creationTime) fo.write("eventPickDataType=%s\n" % eventPickDataType) fo.write("eventPickStreamName=%s\n" % eventPickStreamName) fo.write("eventPickDS=%s\n" % eventPickDS) fo.write("eventPickAmiTag=%s\n" % eventPickAmiTag) fo.write("eventPickNumSites=%s\n" % eventPickNumSites) fo.write("userTaskName=%s\n" % userTaskName) fo.write("userDatasetName=%s\n" % userDatasetName) fo.write("lockedBy=%s\n" % lockedBy) fo.write("params=%s\n" % params) fo.write("inputFileList=%s\n" % inputFileList) for tmpLine in runEventList.split('\n'): tmpItems = tmpLine.split() if len(tmpItems) != 2: continue fo.write("runEvent=%s,%s\n" % tuple(tmpItems)) fo.close() except: errType,errValue = sys.exc_info()[:2] errStr = "cannot put request due to %s %s" % (errType,errValue) _logger.error("putEventPickingRequest : " + errStr + " " + userName) return "ERROR : " + errStr _logger.debug("putEventPickingRequest : %s end" % userName) return True
def touchFile(req, filename): if not Protocol.isSecure(req): return 'False' try: os.utime('%s/%s' % (panda_config.cache_dir, filename.split('/')[-1]), None) return 'True' except: errtype, errvalue = sys.exc_info()[:2] _logger.error("touchFile : %s %s" % (errtype, errvalue)) return 'False'
def uploadLog(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename)) tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN'])) # size check sizeLimit = 100 * 1024 * 1024 # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except: if req.headers_in.has_key("content-length"): tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "failed to upload log due to size limit" tmpLog.error(errStr) tmpLog.debug("end") return errStr jediLogDir = '/jedilog' retStr = '' try: fileBaseName = file.filename.split('/')[-1] fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir, fileBaseName) # delete old file if os.path.exists(fileFullPath): os.remove(fileFullPath) # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() tmpLog.debug("written to {0}".format(fileFullPath)) retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None), jediLogDir, fileBaseName) except: errtype, errvalue = sys.exc_info()[:2] errStr = "failed to write log with {0}:{1}".format( errtype.__name__, errvalue) tmpLog.error(errStr) tmpLog.debug("end") return errStr tmpLog.debug("end") return retStr
def uploadLog(req,file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False tmpLog = LogWrapper(_logger,'uploadLog <{0}>'.format(file.filename)) tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN'])) # size check sizeLimit = 100*1024*1024 # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except: if req.headers_in.has_key("content-length"): tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "failed to upload log due to size limit" tmpLog.error(errStr) tmpLog.debug("end") return errStr jediLogDir = '/jedilog' retStr = '' try: fileBaseName = file.filename.split('/')[-1] fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir,jediLogDir,fileBaseName) # delete old file if os.path.exists(fileFullPath): os.remove(fileFullPath) # write fo = open(fileFullPath,'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() tmpLog.debug("written to {0}".format(fileFullPath)) retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None),jediLogDir,fileBaseName) except: errtype,errvalue = sys.exc_info()[:2] errStr = "failed to write log with {0}:{1}".format(errtype.__name__,errvalue) tmpLog.error(errStr) tmpLog.debug("end") return errStr tmpLog.debug("end") return retStr
def putEventPickingRequest(req, runEventList='', eventPickDataType='', eventPickStreamName='', eventPickDS='', eventPickAmiTag='', userDatasetName='', lockedBy='', params='', inputFileList='', eventPickNumSites='', userTaskName='', ei_api='', giveGUID=None): if not Protocol.isSecure(req): return "ERROR : no HTTPS" userName = req.subprocess_env['SSL_CLIENT_S_DN'] creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') _logger.debug("putEventPickingRequest : %s start" % userName) # size check sizeLimit = 10 * 1024 * 1024 # get total size try: contentLength = long(req.headers_in["content-length"]) except: errStr = "cannot get content-length from HTTP request." _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "Too large run/event list. Exceeded size limit %s>%s." % ( contentLength, sizeLimit) _logger.error("putEventPickingRequest : " + errStr + " " + userName) _logger.debug("putEventPickingRequest : %s end" % userName) return "ERROR : " + errStr if giveGUID == 'True': giveGUID = True else: giveGUID = False try: # make filename evpFileName = '%s/evp.%s' % (panda_config.cache_dir, str(uuid.uuid4())) _logger.debug("putEventPickingRequest : %s -> %s" % (userName, evpFileName)) # write fo = open(evpFileName, 'wb') fo.write("userName=%s\n" % userName) fo.write("creationTime=%s\n" % creationTime) fo.write("eventPickDataType=%s\n" % eventPickDataType) fo.write("eventPickStreamName=%s\n" % eventPickStreamName) fo.write("eventPickDS=%s\n" % eventPickDS) fo.write("eventPickAmiTag=%s\n" % eventPickAmiTag) fo.write("eventPickNumSites=%s\n" % eventPickNumSites) fo.write("userTaskName=%s\n" % userTaskName) fo.write("userDatasetName=%s\n" % userDatasetName) fo.write("lockedBy=%s\n" % lockedBy) fo.write("params=%s\n" % params) fo.write("inputFileList=%s\n" % inputFileList) fo.write("ei_api=%s\n" % ei_api) runEvtGuidMap = {} for tmpLine in runEventList.split('\n'): tmpItems = tmpLine.split() if (len(tmpItems) != 2 and not giveGUID) or \ (len(tmpItems) != 3 and giveGUID): continue fo.write("runEvent=%s,%s\n" % tuple(tmpItems[:2])) if giveGUID: runEvtGuidMap[tuple(tmpItems[:2])] = [tmpItems[2]] fo.write("runEvtGuidMap=%s\n" % str(runEvtGuidMap)) fo.close() except: errType, errValue = sys.exc_info()[:2] errStr = "cannot put request due to %s %s" % (errType, errValue) _logger.error("putEventPickingRequest : " + errStr + " " + userName) return "ERROR : " + errStr _logger.debug("putEventPickingRequest : %s end" % userName) return True
def putFile(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False _logger.debug("putFile : start %s %s" % (req.subprocess_env['SSL_CLIENT_S_DN'], file.filename)) # size check fullSizeLimit = 768 * 1024 * 1024 if not file.filename.startswith('sources.'): noBuild = True sizeLimit = 100 * 1024 * 1024 else: noBuild = False sizeLimit = fullSizeLimit # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except: if req.headers_in.has_key("content-length"): _logger.error("cannot get CL : %s" % req.headers_in["content-length"]) else: _logger.error("no CL") _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % ( contentLength, sizeLimit) if noBuild: errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit" else: errStr += " Please remove redundant files from your workarea" _logger.error(errStr) _logger.debug("putFile : end") return errStr try: fileFullPath = '%s/%s' % (panda_config.cache_dir, file.filename.split('/')[-1]) # avoid overwriting if os.path.exists(fileFullPath): # touch os.utime(fileFullPath, None) # send error message errStr = "ERROR : Cannot overwrite file" _logger.debug('putFile : cannot overwrite file %s' % file.filename) _logger.debug("putFile : end") return errStr # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() except: errStr = "ERROR : Cannot write file" _logger.error(errStr) _logger.debug("putFile : end") return errStr # checksum try: # decode Footer footer = fileContent[-8:] checkSum, isize = struct.unpack("II", footer) _logger.debug("CRC from gzip Footer %s" % checkSum) except: # calculate on the fly """ import zlib checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF """ # use None to avoid delay for now checkSum = None _logger.debug("CRC calculated %s" % checkSum) # file size fileSize = len(fileContent) # user name username = cleanUserID(req.subprocess_env['SSL_CLIENT_S_DN']) _logger.debug("putFile : written dn=%s file=%s size=%s crc=%s" % \ (username,file.filename,fileSize,checkSum)) # put file info to DB statClient, outClient = Client.insertSandboxFileInfo( username, file.filename, fileSize, checkSum) if statClient != 0 or outClient.startswith("ERROR"): _logger.error("putFile : failed to put sandbox to DB with %s %s" % (statClient, outClient)) #_logger.debug("putFile : end") #return "ERROR : Cannot insert sandbox to DB" else: _logger.debug("putFile : inserted sandbox to DB with %s" % outClient) # store to cassandra if hasattr(panda_config, 'cacheUseCassandra') and panda_config.cacheUseCassandra == True: try: # time-stamp timeNow = datetime.datetime.utcnow() creationTime = timeNow.strftime('%Y-%m-%d %H:%M:%S') # user name username = req.subprocess_env['SSL_CLIENT_S_DN'] username = username.replace('/CN=proxy', '') username = username.replace('/CN=limited proxy', '') # file size fileSize = len(fileContent) # key fileKeyName = file.filename.split('/')[-1] sizeCheckSum = '%s:%s' % (fileSize, checkSum) # insert to cassandra import pycassa pool = pycassa.ConnectionPool(panda_config.cacheKeySpace) filefamily = pycassa.ColumnFamily(pool, panda_config.cacheFileTable) # avoid overwriting gotoNextCassa = True if filefamily.get_count(fileKeyName) > 0: # touch touchFlag = touchFileCassa(filefamily, fileKeyName, timeNow) if touchFlag: gotoNextCassa = False # send error message errStr = "ERROR : Cannot overwrite file in Cassandra" _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # check uniqueness with size and checksum if gotoNextCassa: try: uniqExp = pycassa.index.create_index_expression( 'uniqID', sizeCheckSum) userExp = pycassa.index.create_index_expression( 'user', username) tmpClause = pycassa.index.create_index_clause( [uniqExp, userExp]) tmpResults = filefamily.get_indexed_slices( tmpClause, columns=['creationTime']) for oldFileKeyName, tmpDict in tmpResults: _logger.debug('The same size and chksum %s found in old:%s and new:%s' % \ (sizeCheckSum,oldFileKeyName,fileKeyName)) # touch touchFlag = touchFileCassa(filefamily, oldFileKeyName, timeNow) if touchFlag: # make alias _logger.debug('Making alias %s->%s' % (fileKeyName, oldFileKeyName)) insertWithRetryCassa( filefamily, fileKeyName, { 'alias': oldFileKeyName, 'creationTime': creationTime, 'nSplit': 0, }, 'putFile : make alias for %s' % file.filename) # set time touchFileCassa(filefamily, fileKeyName, timeNow) _logger.debug("putFile : end") return True except: gotoNextCassa = False errType, errValue = sys.exc_info()[:2] errStr = "cannot make alias for %s due to %s %s" % ( fileKeyName, errType, errValue) _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # insert new record if gotoNextCassa: splitIdx = 0 splitSize = 5 * 1024 * 1024 nSplit, tmpMod = divmod(len(fileContent), splitSize) if tmpMod != 0: nSplit += 1 _logger.debug('Inserting %s with %s blocks' % (fileKeyName, nSplit)) for splitIdx in range(nSplit): # split to small chunks since cassandra is not good at large files tmpFileContent = fileContent[splitSize * splitIdx:splitSize * (splitIdx + 1)] tmpFileKeyName = fileKeyName tmpAttMap = { 'file': tmpFileContent, 'user': username, 'creationTime': creationTime, } if splitIdx == 0: tmpAttMap['size'] = fileSize tmpAttMap['nSplit'] = nSplit tmpAttMap['uniqID'] = sizeCheckSum tmpAttMap['checkSum'] = str(checkSum) else: tmpFileKeyName += '_%s' % splitIdx tmpAttMap['size'] = 0 tmpAttMap['nSplit'] = 0 # insert with retry insertWithRetryCassa(filefamily, tmpFileKeyName, tmpAttMap, 'putFile : insert %s' % file.filename) # set time touchFileCassa(filefamily, fileKeyName, timeNow) except: errType, errValue = sys.exc_info()[:2] errStr = "cannot put %s into Cassandra due to %s %s" % ( fileKeyName, errType, errValue) _logger.error(errStr) # send error message errStr = "ERROR : " + errStr if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr _logger.debug("putFile : %s end" % file.filename) return True
def putFile(req,file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False _logger.debug("putFile : start %s %s" % (req.subprocess_env['SSL_CLIENT_S_DN'],file.filename)) # size check fullSizeLimit = 768*1024*1024 if not file.filename.startswith('sources.'): noBuild = True sizeLimit = 100*1024*1024 else: noBuild = False sizeLimit = fullSizeLimit # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except: if req.headers_in.has_key("content-length"): _logger.error("cannot get CL : %s" % req.headers_in["content-length"]) else: _logger.error("no CL") _logger.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % (contentLength,sizeLimit) if noBuild: errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit" else: errStr += " Please remove redundant files from your workarea" _logger.error(errStr) _logger.debug("putFile : end") return errStr try: fileFullPath = '%s/%s' % (panda_config.cache_dir,file.filename.split('/')[-1]) # avoid overwriting if os.path.exists(fileFullPath): # touch os.utime(fileFullPath,None) # send error message errStr = "ERROR : Cannot overwrite file" _logger.debug('putFile : cannot overwrite file %s' % file.filename) _logger.debug("putFile : end") return errStr # write fo = open(fileFullPath,'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() except: errStr = "ERROR : Cannot write file" _logger.error(errStr) _logger.debug("putFile : end") return errStr # checksum try: # decode Footer footer = fileContent[-8:] checkSum,isize = struct.unpack("II",footer) _logger.debug("CRC from gzip Footer %s" % checkSum) except: # calculate on the fly """ import zlib checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF """ # use None to avoid delay for now checkSum = None _logger.debug("CRC calculated %s" % checkSum) # file size fileSize = len(fileContent) # user name username = cleanUserID(req.subprocess_env['SSL_CLIENT_S_DN']) _logger.debug("putFile : written dn=%s file=%s size=%s crc=%s" % \ (username,file.filename,fileSize,checkSum)) # put file info to DB statClient,outClient = Client.insertSandboxFileInfo(username,file.filename, fileSize,checkSum) if statClient != 0 or outClient.startswith("ERROR"): _logger.error("putFile : failed to put sandbox to DB with %s %s" % (statClient,outClient)) #_logger.debug("putFile : end") #return "ERROR : Cannot insert sandbox to DB" else: _logger.debug("putFile : inserted sandbox to DB with %s" % outClient) # store to cassandra if hasattr(panda_config,'cacheUseCassandra') and panda_config.cacheUseCassandra == True: try: # time-stamp timeNow = datetime.datetime.utcnow() creationTime = timeNow.strftime('%Y-%m-%d %H:%M:%S') # user name username = req.subprocess_env['SSL_CLIENT_S_DN'] username = username.replace('/CN=proxy','') username = username.replace('/CN=limited proxy','') # file size fileSize = len(fileContent) # key fileKeyName = file.filename.split('/')[-1] sizeCheckSum = '%s:%s' % (fileSize,checkSum) # insert to cassandra import pycassa pool = pycassa.ConnectionPool(panda_config.cacheKeySpace) filefamily = pycassa.ColumnFamily(pool,panda_config.cacheFileTable) # avoid overwriting gotoNextCassa = True if filefamily.get_count(fileKeyName) > 0: # touch touchFlag = touchFileCassa(filefamily,fileKeyName,timeNow) if touchFlag: gotoNextCassa = False # send error message errStr = "ERROR : Cannot overwrite file in Cassandra" _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # check uniqueness with size and checksum if gotoNextCassa: try: uniqExp = pycassa.index.create_index_expression('uniqID',sizeCheckSum) userExp = pycassa.index.create_index_expression('user',username) tmpClause = pycassa.index.create_index_clause([uniqExp,userExp]) tmpResults = filefamily.get_indexed_slices(tmpClause,columns=['creationTime']) for oldFileKeyName,tmpDict in tmpResults: _logger.debug('The same size and chksum %s found in old:%s and new:%s' % \ (sizeCheckSum,oldFileKeyName,fileKeyName)) # touch touchFlag = touchFileCassa(filefamily,oldFileKeyName,timeNow) if touchFlag: # make alias _logger.debug('Making alias %s->%s' % (fileKeyName,oldFileKeyName)) insertWithRetryCassa(filefamily,fileKeyName, {'alias':oldFileKeyName, 'creationTime':creationTime, 'nSplit':0, }, 'putFile : make alias for %s' % file.filename ) # set time touchFileCassa(filefamily,fileKeyName,timeNow) _logger.debug("putFile : end") return True except: gotoNextCassa = False errType,errValue = sys.exc_info()[:2] errStr = "cannot make alias for %s due to %s %s" % (fileKeyName,errType,errValue) _logger.error(errStr) if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr # insert new record if gotoNextCassa: splitIdx = 0 splitSize = 5 * 1024 * 1024 nSplit,tmpMod = divmod(len(fileContent),splitSize) if tmpMod != 0: nSplit += 1 _logger.debug('Inserting %s with %s blocks' % (fileKeyName,nSplit)) for splitIdx in range(nSplit): # split to small chunks since cassandra is not good at large files tmpFileContent = fileContent[splitSize*splitIdx:splitSize*(splitIdx+1)] tmpFileKeyName = fileKeyName tmpAttMap = {'file':tmpFileContent, 'user':username, 'creationTime':creationTime, } if splitIdx == 0: tmpAttMap['size'] = fileSize tmpAttMap['nSplit'] = nSplit tmpAttMap['uniqID'] = sizeCheckSum tmpAttMap['checkSum'] = str(checkSum) else: tmpFileKeyName += '_%s' % splitIdx tmpAttMap['size'] = 0 tmpAttMap['nSplit'] = 0 # insert with retry insertWithRetryCassa(filefamily,tmpFileKeyName,tmpAttMap, 'putFile : insert %s' % file.filename) # set time touchFileCassa(filefamily,fileKeyName,timeNow) except: errType,errValue = sys.exc_info()[:2] errStr = "cannot put %s into Cassandra due to %s %s" % (fileKeyName,errType,errValue) _logger.error(errStr) # send error message errStr = "ERROR : " + errStr if not panda_config.cacheIgnoreCassandraError: _logger.debug("putFile : end") return errStr _logger.debug("putFile : %s end" % file.filename) return True