def _purgeThread(srvObj, reqPropsObj, httpRef):
global is_purgeThrd_running, total_todo, num_done
is_purgeThrd_running = True
try:
logger.debug("host_id = %s", srvObj.getHostId())
resDel = srvObj.getDb().query2(QUERY_ALL_FILES, args=(srvObj.getHostId(),))
if not resDel:
raise Exception('Could not find any files to discard / retain')
else:
total_todo = len(resDel)
for fileDelInfo in resDel:
try:
if (_shouldRetain(fileDelInfo[1])):
continue
ngamsDiscardCmd._discardFile(srvObj, fileDelInfo[0], fileDelInfo[1], int(fileDelInfo[2]), execute = 1)
num_done += 1
except Exception as e1:
if (str(e1).find('DISCARD Command can only be executed locally') > -1):
#warning(str(e1))
continue
else:
raise e1
except Exception:
errMsg = 'Fail to execute the rri purge thread'
logger.exception(errMsg)
finally:
is_purgeThrd_running = False
total_todo = 0
num_done = 0
def _purgeThread(srvObj, reqPropsObj, httpRef):
global is_purgeThrd_running, total_todo, num_done
is_purgeThrd_running = True
hostId = srvObj.getHostId()
try:
if (reqPropsObj.hasHttpPar("keep_earliest")
): # early could be 1, or 2,...
if (reqPropsObj.hasHttpPar("pv_on_any_hosts")):
resDel = srvObj.getDb().query2(
QUERY_LATER_VER_POAH,
args=(hostId,
)) # grab all later versions on this host to remove
else:
resDel = srvObj.getDb().query2(
QUERY_LATER_VER,
args=(hostId, hostId
)) # grab all later versions on this host to remove
else: # by default, keep latest
resDel = srvObj.getDb().query2(
QUERY_PREV_VER,
args=(hostId, hostId)) # grab all previous versions to remove
if not resDel:
raise Exception('Could not find any files to discard / retain')
else:
total_todo = len(resDel)
for fileDelInfo in resDel:
try:
ngamsDiscardCmd._discardFile(srvObj,
fileDelInfo[0],
fileDelInfo[1],
int(fileDelInfo[2]),
execute=1)
num_done += 1
except Exception as e1:
if (str(e1).find(
'DISCARD Command can only be executed locally') >
-1):
#warning(str(e1))
continue
else:
raise e1
except Exception:
errMsg = 'Fail to execute the retainThread'
logger.exception(errMsg)
finally:
is_purgeThrd_running = False
total_todo = 0
num_done = 0
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Compress the file based on file_path (file_id, disk_id, and file_version)
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# target_host does not contain port
required_params = [
'file_path', 'file_id', 'file_version', 'disk_id', 'target_host',
'crc_db', 'debug'
]
filename = reqPropsObj.getHttpPar('file_path')
if (not os.path.exists(filename)):
errMsg = 'File does not exist: %s' % filename
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=501)
return
parDic = reqPropsObj.getHttpParsDic()
for rp in required_params:
if (not parDic.has_key(rp)):
errMsg = 'Parameter missing: %s' % rp
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=502)
return
logger.debug('Moving file %s', filename)
fileId = parDic['file_id']
fileVersion = int(parDic['file_version'])
diskId = parDic['disk_id']
tgtHost = parDic['target_host']
crcDb = parDic['crc_db']
debug = int(parDic['debug'])
if (not isMWAVisFile(fileId)):
errMsg = 'Not MWA visibilty file: %' % (fileId)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=503)
return
"""
if (fileOnHost(srvObj, fileId, tgtHost)):
errMsg = "File %s already on host %s" % (fileId, tgtHost)
warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT) # make it correct
return
"""
if (debug):
logger.debug('Only for file movment debugging, return now')
httpRef.send_data('DEBUG ONLY', NGAMS_TEXT_MT)
return
fileCRC = getCRCFromFile(filename)
if (fileCRC != crcDb):
errMsg = 'File %s on source host %s already corrupted, moving request rejected' % (
filename, srvObj.getHostId().replace(':', '-'))
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504)
return
sendUrl = 'http://%s:7777/QARCHIVE' % tgtHost
logger.debug("Moving to %s", sendUrl)
fileMimeType = 'application/octet-stream'
baseName = os.path.basename(filename)
contDisp = "attachment; filename=\"" + baseName + "\""
deliver_success = False
last_deliv_err = ''
stat = ngamsStatus.ngamsStatus()
hdrs = {NGAMS_HTTP_HDR_CHECKSUM: fileCRC}
for i in range(3): # total trials - 3 times
with open(filename, "rb") as f:
try:
reply, msg, hdrs, data = ngamsHttpUtils.httpPostUrl(
sendUrl, f, fileMimeType, contDisp=contDisp, hdrs=hdrs)
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
if (reply != NGAMS_HTTP_SUCCESS
or stat.getStatus() == NGAMS_FAILURE):
logger.warning("Attempt %d failed: %s", i,
stat.getMessage())
last_deliv_err = stat.getMessage()
continue
else:
deliver_success = True
break
except Exception as hexp:
logger.warning("Attempt %d failed: %s", i, str(hexp))
last_deliv_err = str(hexp).replace('\n', '--')
continue
if (not deliver_success):
errMsg = 'File %s failed to be moved to %s: %s' % (fileId, tgtHost,
last_deliv_err)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=505)
return
try:
ngamsDiscardCmd._discardFile(srvObj,
diskId,
fileId,
fileVersion,
execute=1)
except Exception as e1:
logger.warning('Fail to remove file %s: %s', filename, str(e1))
httpRef.send_data('Remove error: %s' % str(e1).replace('\n', '--'),
NGAMS_TEXT_MT)
return
httpRef.send_data(MOVE_SUCCESS, NGAMS_TEXT_MT)
hdrs=hdrs)
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
if (reply != NGAMS_HTTP_SUCCESS or stat.getStatus() == NGAMS_FAILURE):
logger.warning("Attempt %d failed: %s", i, stat.getMessage())
last_deliv_err = stat.getMessage()
continue
else:
deliver_success = True
break
except Exception, hexp:
logger.warning("Attempt %d failed: %s", i, str(hexp))
last_deliv_err = str(hexp).replace('\n', '--')
continue
if (not deliver_success):
errMsg = 'File %s failed to be moved to %s: %s' % (fileId, tgtHost, last_deliv_err)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=505)
return
try:
ngamsDiscardCmd._discardFile(srvObj, diskId, fileId, fileVersion, execute = 1)
except Exception, e1:
logger.warning('Fail to remove file %s: %s', filename, str(e1))
httpRef.send_data('Remove error: %s' % str(e1).replace('\n', '--'), NGAMS_TEXT_MT)
return
httpRef.send_data(MOVE_SUCCESS, NGAMS_TEXT_MT)