def put_file_recovery_request(req, jediTaskID, dryRun=None):
if not Protocol.isSecure(req):
return json.dumps((False, "ERROR : no HTTPS"))
userName = req.subprocess_env['SSL_CLIENT_S_DN']
creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
tmpLog = LogWrapper(_logger, 'put_file_recovery_request < jediTaskID={}'.format(jediTaskID))
tmpLog.debug("start user={}".format(userName))
# get total size
try:
jediTaskID = int(jediTaskID)
# make filename
evpFileName = '%s/recov.%s' % (panda_config.cache_dir,str(uuid.uuid4()))
tmpLog.debug("file={}".format(evpFileName))
# write
with open(evpFileName, 'w') as fo:
data = {"userName": userName,
"creationTime": creationTime,
"jediTaskID": int(jediTaskID)
}
if dryRun:
data['dryRun'] = True
json.dump(data, fo)
except Exception as e:
errStr = "cannot put request due to {} ".format(str(e))
tmpLog.error(errStr + traceback.format_exc())
return json.dumps((False, errStr))
tmpLog.debug('done')
return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def deleteFile(req, file):
if not Protocol.isSecure(req):
return 'False'
try:
# may be reused for rebrokreage
#os.remove('%s/%s' % (panda_config.cache_dir,file.split('/')[-1]))
return 'True'
except Exception:
return 'False'
def touchFile(req,filename):
if not Protocol.isSecure(req):
return 'False'
try:
os.utime('%s/%s' % (panda_config.cache_dir,filename.split('/')[-1]),None)
return 'True'
except Exception:
errtype,errvalue = sys.exc_info()[:2]
_logger.error("touchFile : %s %s" % (errtype,errvalue))
return 'False'
def uploadLog(req, file):
if not Protocol.isSecure(req):
return False
if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']:
return False
tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename))
tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN']))
# size check
sizeLimit = 100 * 1024 * 1024
# get file size
contentLength = 0
try:
contentLength = long(req.headers_in["content-length"])
except Exception:
if "content-length" in req.headers_in:
tmpLog.error("cannot get CL : %s" %
req.headers_in["content-length"])
else:
tmpLog.error("no CL")
tmpLog.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "failed to upload log due to size limit"
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
jediLogDir = '/jedilog'
retStr = ''
try:
fileBaseName = file.filename.split('/')[-1]
fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir,
fileBaseName)
# delete old file
if os.path.exists(fileFullPath):
os.remove(fileFullPath)
# write
fo = open(fileFullPath, 'wb')
fileContent = file.file.read()
fo.write(fileContent)
fo.close()
tmpLog.debug("written to {0}".format(fileFullPath))
retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None),
jediLogDir, fileBaseName)
except Exception:
errtype, errvalue = sys.exc_info()[:2]
errStr = "failed to write log with {0}:{1}".format(
errtype.__name__, errvalue)
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
tmpLog.debug("end")
return retStr
def delete_checkpoint(req, task_id, sub_id):
tmpLog = LogWrapper(_logger, 'delete_checkpoint <jediTaskID={0} ID={1}>'.format(task_id, sub_id))
status = True
if not Protocol.isSecure(req):
msg = 'insecure request'
tmpLog.error(msg)
status = False
else:
tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN'])
try:
fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id))
os.remove(fileFullPath)
msg = 'done'
tmpLog.debug(msg)
except Exception as e:
msg = "failed to delete file due to {0}".format(str(e))
tmpLog.error(msg)
status = False
return json.dumps({'status': status, 'message': msg})
def put_checkpoint(req, file):
tmpLog = LogWrapper(_logger, 'put_checkpoint <jediTaskID_subID={0}>'.format(file.filename))
status = False
if not Protocol.isSecure(req):
errStr = 'insecure request'
tmpLog.error(errStr)
return json.dumps({'status': status, 'message': errStr})
tmpLog.debug("start %s" % req.subprocess_env['SSL_CLIENT_S_DN'])
# extract taskID and subID
try:
task_id, sub_id = file.filename.split('/')[-1].split('_')
except Exception:
errStr = 'failed to extract ID'
tmpLog.error(errStr)
return json.dumps({'status': status, 'message': errStr})
# size check
sizeLimit = 500 * 1024 * 1024
# get file size
try:
contentLength = long(req.headers_in["content-length"])
except Exception as e:
errStr = "cannot get int(content-length) due to {0}".format(str(e))
tmpLog.error(errStr)
return json.dumps({'status': status, 'message': errStr})
tmpLog.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "exceeded size limit %s>%s" % (contentLength, sizeLimit)
tmpLog.error(errStr)
return json.dumps({'status': status, 'message': errStr})
try:
fileFullPath = os.path.join(panda_config.cache_dir, get_checkpoint_filename(task_id, sub_id))
# write
with open(fileFullPath,'wb') as fo:
fo.write(file.file.read())
except Exception as e:
errStr = "cannot write file due to {0}".format(str(e))
tmpLog.error(errStr)
return json.dumps({'status': status, 'message': errStr})
status = True
tmpMsg = "successfully placed at {0}".format(fileFullPath)
tmpLog.debug(tmpMsg)
return json.dumps({'status': status, 'message': tmpMsg})
def put_workflow_request(req, data, check=False):
if not Protocol.isSecure(req):
return json.dumps((False, "ERROR : no HTTPS"))
userName = req.subprocess_env['SSL_CLIENT_S_DN']
creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
tmpLog = LogWrapper(_logger, 'put_workflow_request')
tmpLog.debug("start user={} check={}".format(userName, check))
if check == 'True' or check is True:
check = True
else:
check = False
# get total size
try:
# make filename
evpFileName = '%s/workflow.%s' % (panda_config.cache_dir,str(uuid.uuid4()))
tmpLog.debug("file={}".format(evpFileName))
# write
with open(evpFileName, 'w') as fo:
data = {"userName": userName,
"creationTime": creationTime,
"data": json.loads(data),
}
json.dump(data, fo)
# check
if check:
tmpLog.debug('checking')
from pandaserver.taskbuffer.workflow_processor import WorkflowProcessor
processor = WorkflowProcessor(log_stream=_logger)
ret = processor.process(evpFileName, True, True, True, True)
if os.path.exists(evpFileName):
try:
os.remove(evpFileName)
except Exception:
pass
tmpLog.debug('done')
return json.dumps((True, ret))
except Exception as e:
errStr = "cannot put request due to {} ".format(str(e))
tmpLog.error(errStr + traceback.format_exc())
return json.dumps((False, errStr))
tmpLog.debug('done')
return json.dumps((True, 'request was accepted and will be processed in a few minutes'))
def putEventPickingRequest(req,
runEventList='',
eventPickDataType='',
eventPickStreamName='',
eventPickDS='',
eventPickAmiTag='',
userDatasetName='',
lockedBy='',
params='',
inputFileList='',
eventPickNumSites='',
userTaskName='',
ei_api='',
giveGUID=None):
if not Protocol.isSecure(req):
return "ERROR : no HTTPS"
userName = req.subprocess_env['SSL_CLIENT_S_DN']
creationTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
_logger.debug("putEventPickingRequest : %s start" % userName)
# size check
sizeLimit = 10 * 1024 * 1024
# get total size
try:
contentLength = long(req.headers_in["content-length"])
except Exception:
errStr = "cannot get content-length from HTTP request."
_logger.error("putEventPickingRequest : " + errStr + " " + userName)
_logger.debug("putEventPickingRequest : %s end" % userName)
return "ERROR : " + errStr
_logger.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "Too large run/event list. Exceeded size limit %s>%s." % (
contentLength, sizeLimit)
_logger.error("putEventPickingRequest : " + errStr + " " + userName)
_logger.debug("putEventPickingRequest : %s end" % userName)
return "ERROR : " + errStr
if giveGUID == 'True':
giveGUID = True
else:
giveGUID = False
try:
# make filename
evpFileName = '%s/evp.%s' % (panda_config.cache_dir, str(uuid.uuid4()))
_logger.debug("putEventPickingRequest : %s -> %s" %
(userName, evpFileName))
# write
fo = open(evpFileName, 'wb')
fo.write("userName=%s\n" % userName)
fo.write("creationTime=%s\n" % creationTime)
fo.write("eventPickDataType=%s\n" % eventPickDataType)
fo.write("eventPickStreamName=%s\n" % eventPickStreamName)
fo.write("eventPickDS=%s\n" % eventPickDS)
fo.write("eventPickAmiTag=%s\n" % eventPickAmiTag)
fo.write("eventPickNumSites=%s\n" % eventPickNumSites)
fo.write("userTaskName=%s\n" % userTaskName)
fo.write("userDatasetName=%s\n" % userDatasetName)
fo.write("lockedBy=%s\n" % lockedBy)
fo.write("params=%s\n" % params)
fo.write("inputFileList=%s\n" % inputFileList)
fo.write("ei_api=%s\n" % ei_api)
runEvtGuidMap = {}
for tmpLine in runEventList.split('\n'):
tmpItems = tmpLine.split()
if (len(tmpItems) != 2 and not giveGUID) or \
(len(tmpItems) != 3 and giveGUID):
continue
fo.write("runEvent=%s,%s\n" % tuple(tmpItems[:2]))
if giveGUID:
runEvtGuidMap[tuple(tmpItems[:2])] = [tmpItems[2]]
fo.write("runEvtGuidMap=%s\n" % str(runEvtGuidMap))
fo.close()
except Exception:
errType, errValue = sys.exc_info()[:2]
errStr = "cannot put request due to %s %s" % (errType, errValue)
_logger.error("putEventPickingRequest : " + errStr + " " + userName)
return "ERROR : " + errStr
_logger.debug("putEventPickingRequest : %s end" % userName)
return True
def putFile(req, file):
if not Protocol.isSecure(req):
return False
if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']:
return False
_logger.debug("putFile : start %s %s" %
(req.subprocess_env['SSL_CLIENT_S_DN'], file.filename))
# size check
fullSizeLimit = 768 * 1024 * 1024
if not file.filename.startswith('sources.'):
noBuild = True
sizeLimit = 100 * 1024 * 1024
else:
noBuild = False
sizeLimit = fullSizeLimit
# get file size
contentLength = 0
try:
contentLength = long(req.headers_in["content-length"])
except Exception:
if "content-length" in req.headers_in:
_logger.error("cannot get CL : %s" %
req.headers_in["content-length"])
else:
_logger.error("no CL")
_logger.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % (
contentLength, sizeLimit)
if noBuild:
errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit"
else:
errStr += " Please remove redundant files from your workarea"
_logger.error(errStr)
_logger.debug("putFile : end")
return errStr
try:
fileFullPath = '%s/%s' % (panda_config.cache_dir,
file.filename.split('/')[-1])
# avoid overwriting
if os.path.exists(fileFullPath):
# touch
os.utime(fileFullPath, None)
# send error message
errStr = "ERROR : Cannot overwrite file"
_logger.debug('putFile : cannot overwrite file %s' % file.filename)
_logger.debug("putFile : end")
return errStr
# write
fo = open(fileFullPath, 'wb')
fileContent = file.file.read()
fo.write(fileContent)
fo.close()
except Exception:
errStr = "ERROR : Cannot write file"
_logger.error(errStr)
_logger.debug("putFile : end")
return errStr
# checksum
try:
# decode Footer
footer = fileContent[-8:]
checkSum, isize = struct.unpack("II", footer)
_logger.debug("CRC from gzip Footer %s" % checkSum)
except Exception:
# calculate on the fly
"""
import zlib
checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF
"""
# use None to avoid delay for now
checkSum = None
_logger.debug("CRC calculated %s" % checkSum)
# file size
fileSize = len(fileContent)
# user name
username = cleanUserID(req.subprocess_env['SSL_CLIENT_S_DN'])
_logger.debug("putFile : written dn=%s file=%s size=%s crc=%s" % \
(username,file.filename,fileSize,checkSum))
# put file info to DB
statClient, outClient = Client.insertSandboxFileInfo(
username, file.filename, fileSize, checkSum)
if statClient != 0 or outClient.startswith("ERROR"):
_logger.error("putFile : failed to put sandbox to DB with %s %s" %
(statClient, outClient))
#_logger.debug("putFile : end")
#return "ERROR : Cannot insert sandbox to DB"
else:
_logger.debug("putFile : inserted sandbox to DB with %s" % outClient)
# store to cassandra
if hasattr(panda_config,
'cacheUseCassandra') and panda_config.cacheUseCassandra == True:
try:
# time-stamp
timeNow = datetime.datetime.utcnow()
creationTime = timeNow.strftime('%Y-%m-%d %H:%M:%S')
# user name
username = req.subprocess_env['SSL_CLIENT_S_DN']
username = username.replace('/CN=proxy', '')
username = username.replace('/CN=limited proxy', '')
# file size
fileSize = len(fileContent)
# key
fileKeyName = file.filename.split('/')[-1]
sizeCheckSum = '%s:%s' % (fileSize, checkSum)
# insert to cassandra
import pycassa
pool = pycassa.ConnectionPool(panda_config.cacheKeySpace)
filefamily = pycassa.ColumnFamily(pool,
panda_config.cacheFileTable)
# avoid overwriting
gotoNextCassa = True
if filefamily.get_count(fileKeyName) > 0:
# touch
touchFlag = touchFileCassa(filefamily, fileKeyName, timeNow)
if touchFlag:
gotoNextCassa = False
# send error message
errStr = "ERROR : Cannot overwrite file in Cassandra"
_logger.error(errStr)
if not panda_config.cacheIgnoreCassandraError:
_logger.debug("putFile : end")
return errStr
# check uniqueness with size and checksum
if gotoNextCassa:
try:
uniqExp = pycassa.index.create_index_expression(
'uniqID', sizeCheckSum)
userExp = pycassa.index.create_index_expression(
'user', username)
tmpClause = pycassa.index.create_index_clause(
[uniqExp, userExp])
tmpResults = filefamily.get_indexed_slices(
tmpClause, columns=['creationTime'])
for oldFileKeyName, tmpDict in tmpResults:
_logger.debug('The same size and chksum %s found in old:%s and new:%s' % \
(sizeCheckSum,oldFileKeyName,fileKeyName))
# touch
touchFlag = touchFileCassa(filefamily, oldFileKeyName,
timeNow)
if touchFlag:
# make alias
_logger.debug('Making alias %s->%s' %
(fileKeyName, oldFileKeyName))
insertWithRetryCassa(
filefamily, fileKeyName, {
'alias': oldFileKeyName,
'creationTime': creationTime,
'nSplit': 0,
},
'putFile : make alias for %s' % file.filename)
# set time
touchFileCassa(filefamily, fileKeyName, timeNow)
_logger.debug("putFile : end")
return True
except Exception:
gotoNextCassa = False
errType, errValue = sys.exc_info()[:2]
errStr = "cannot make alias for %s due to %s %s" % (
fileKeyName, errType, errValue)
_logger.error(errStr)
if not panda_config.cacheIgnoreCassandraError:
_logger.debug("putFile : end")
return errStr
# insert new record
if gotoNextCassa:
splitIdx = 0
splitSize = 5 * 1024 * 1024
nSplit, tmpMod = divmod(len(fileContent), splitSize)
if tmpMod != 0:
nSplit += 1
_logger.debug('Inserting %s with %s blocks' %
(fileKeyName, nSplit))
for splitIdx in range(nSplit):
# split to small chunks since cassandra is not good at large files
tmpFileContent = fileContent[splitSize *
splitIdx:splitSize *
(splitIdx + 1)]
tmpFileKeyName = fileKeyName
tmpAttMap = {
'file': tmpFileContent,
'user': username,
'creationTime': creationTime,
}
if splitIdx == 0:
tmpAttMap['size'] = fileSize
tmpAttMap['nSplit'] = nSplit
tmpAttMap['uniqID'] = sizeCheckSum
tmpAttMap['checkSum'] = str(checkSum)
else:
tmpFileKeyName += '_%s' % splitIdx
tmpAttMap['size'] = 0
tmpAttMap['nSplit'] = 0
# insert with retry
insertWithRetryCassa(filefamily, tmpFileKeyName, tmpAttMap,
'putFile : insert %s' % file.filename)
# set time
touchFileCassa(filefamily, fileKeyName, timeNow)
except Exception:
errType, errValue = sys.exc_info()[:2]
errStr = "cannot put %s into Cassandra due to %s %s" % (
fileKeyName, errType, errValue)
_logger.error(errStr)
# send error message
errStr = "ERROR : " + errStr
if not panda_config.cacheIgnoreCassandraError:
_logger.debug("putFile : end")
return errStr
_logger.debug("putFile : %s end" % file.filename)
return True
def putFile(req, file):
tmpLog = LogWrapper(_logger, 'putFile-{}'.format(datetime.datetime.utcnow().isoformat('/')))
if not Protocol.isSecure(req):
tmpLog.error('No SSL_CLIENT_S_DN')
return False
if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']:
return False
# user name
username = CoreUtils.clean_user_id(req.subprocess_env['SSL_CLIENT_S_DN'])
tmpLog.debug("start %s %s" % (username, file.filename))
# size check
fullSizeLimit = 768*1024*1024
if not file.filename.startswith('sources.'):
noBuild = True
sizeLimit = 100*1024*1024
else:
noBuild = False
sizeLimit = fullSizeLimit
# get file size
contentLength = 0
try:
contentLength = long(req.headers_in["content-length"])
except Exception:
if "content-length" in req.headers_in:
tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"])
else:
tmpLog.error("no CL")
tmpLog.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "ERROR : Upload failure. Exceeded size limit %s>%s." % (contentLength,sizeLimit)
if noBuild:
errStr += " Please submit the job without --noBuild/--libDS since those options impose a tighter size limit"
else:
errStr += " Please remove redundant files from your workarea"
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
try:
fileName = file.filename.split('/')[-1]
fileFullPath = '%s/%s' % (panda_config.cache_dir, fileName)
# avoid overwriting
if os.path.exists(fileFullPath):
# touch
os.utime(fileFullPath,None)
# send error message
errStr = "ERROR : Cannot overwrite file"
tmpLog.debug('cannot overwrite file %s' % fileName)
tmpLog.debug("end")
return errStr
# write
fo = open(fileFullPath,'wb')
fileContent = file.file.read()
if hasattr(panda_config, 'compress_file_names') and \
[True for patt in panda_config.compress_file_names.split(',') if re.search(patt, fileName) is not None]:
fileContent = gzip.compress(fileContent)
fo.write(fileContent)
fo.close()
except Exception:
errStr = "ERROR : Cannot write file"
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
# checksum
try:
# decode Footer
footer = fileContent[-8:]
checkSum,isize = struct.unpack("II",footer)
tmpLog.debug("CRC from gzip Footer %s" % checkSum)
except Exception:
# calculate on the fly
"""
import zlib
checkSum = zlib.adler32(fileContent) & 0xFFFFFFFF
"""
# use None to avoid delay for now
checkSum = None
tmpLog.debug("CRC calculated %s" % checkSum)
# file size
fileSize = len(fileContent)
tmpLog.debug("written dn=%s file=%s size=%s crc=%s" % \
(username, fileFullPath, fileSize, checkSum))
# put file info to DB
if panda_config.record_sandbox_info:
to_insert = True
for patt in IGNORED_SUFFIX:
if file.filename.endswith(patt):
to_insert = False
break
if not to_insert:
tmpLog.debug("skipped to insert to DB")
else:
statClient,outClient = Client.insertSandboxFileInfo(username,file.filename,
fileSize,checkSum)
if statClient != 0 or outClient.startswith("ERROR"):
tmpLog.error("failed to put sandbox to DB with %s %s" % (statClient,outClient))
#_logger.debug("putFile : end")
#return "ERROR : Cannot insert sandbox to DB"
else:
tmpLog.debug("inserted sandbox to DB with %s" % outClient)
tmpLog.debug("end")
return True