def bbcp_transfer(request, out_fname, crc_name, skip_crc):
# source_file must have the IP of the client, so bbcp can contact it and
# launch a copy ot itself.
# OTOH, target_file must have the IP through which we received the original
# HTTP request to ensure the same network path is used by bbcp
bparam = get_params(request)
source_file = get_source_file(request)
target_file = request.getHttpHdr('host') + ':' + out_fname
# Make target file writable if existing.
if (os.path.exists(out_fname)):
os.chmod(out_fname, 420)
checkCreatePath(os.path.dirname(out_fname))
# perform the bbcp transfer, we will always return the checksum
start = time.time()
checksum = bbcpFile(source_file, target_file, bparam, crc_name, skip_crc)
size = getFileSize(out_fname)
totaltime = time.time() - start
# Feedback the file size into the request object now that we know it,
# just in case we need it later
request.setSize(size)
# Artificially split the total time between read and write (+ 0 crc),
# we don't actually know how much was spent on what
half = totaltime / 2
return ngamsArchiveUtils.archiving_results(size, half, half, 0, totaltime,
crc_name, checksum)
def finfo_from_database(fileInfo, srvObj, reqPropsObj):
fileId = fileInfo.getFileId()
fileVer = fileInfo.getFileVersion()
# Locate the file best suiting the query and send it back if possible.
location, _, ipAddress, port, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId,
fileVersion=fileVer,
reqPropsObj=reqPropsObj)
basename = os.path.basename(filename)
if location == NGAMS_HOST_LOCAL:
absname = os.path.normpath(os.path.join(mountPoint, filename))
size = getFileSize(absname)
opener = fopener(absname)
elif location == NGAMS_HOST_CLUSTER or location == NGAMS_HOST_REMOTE:
size = srvObj.getDb().getFileSize(fileId, fileVersion)
opener = http_opener(ipAddress, port, fileId, fileVersion, srvObj)
else:
raise Exception("Unknown location type: %s" % (location, ))
return ngamsMIMEMultipart.file_info(mimeType, basename, size, opener)
def ngamsGLEAM_Fix_Phase2_JobPI(srvObj, plugInPars, filename, fileId,
fileVersion, diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
date_obs = pyfits.getval(filename, 'DATE-OBS').split('T')[0]
if (not dict_dec.has_key(date_obs)):
return (1, 'no date for %s' % filename)
# copy it to some tmp directory
fndir = os.path.dirname(filename)
bname = os.path.basename(filename)
execCmd('cp %s %s/' % (filename, work_dir))
# add header keyword
fn = '%s/%s' % (work_dir, bname)
os.chmod(fn, 644) # owner writable
hdulist = pyfits.open(fn, mode='update')
prihdr = hdulist[0].header
prihdr['DEC_PNT'] = dict_dec[date_obs]
hdulist.close()
# double check if the key is really added or the file is not corrupted
if 'DEC_PNT' not in pyfits.getheader(filename):
# header is not added
# remove the temp file
os.remove(fn)
# raise error
raise Exception('%s still does not have key DEC_PNT' % (fn))
# calculate the checksum, new filesize
csum = getFileCRC(fn)
fsize = getFileSize(fn)
# move the original file to a different name
os.chmod(filename, 644)
execCmd('mv %s %s/%s_origin' % (filename, fndir, bname))
# move the new file back under the original name
execCmd('mv %s %s/' % (fn, fndir))
os.chmod(filename, 444) # make it readonly
# if all fine, remove the original file (under the different name)
# otherwise, remove the new file, and move the original file back to the original name
os.remove('%s/%s_origin' % (fndir, bname))
def _checkFile(srvObj,
fileInfoObj,
stagFile,
httpHeaders,
checkChecksum):
"""
Make a consistency check of the Staging File.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
(ngamsFileInfo).
stagFile: Staging filename (string).
httpHeaders: HTTP headers (mimetools.Message)
checkChecksum: Carry out checksum check (0|1/integer).
Returns: Void.
"""
# First ensure to flush file caches.
ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagFile])
# Check file size.
fileSize = getFileSize(stagFile)
if (fileSize != fileInfoObj.getFileSize()):
# If the mime-type is 'text/xml' we check if the returned
# document is an NG/AMS XML Status reporting a problem.
tmpStat = None
if (httpHeaders.type == NGAMS_XML_MT):
try:
tmpStat = ngamsStatus.ngamsStatus().load(stagFile,getStatus=1)
if (tmpStat.getStatus() != NGAMS_FAILURE):
del tmpStat
tmpStat = None
except:
# Was apparently not an NG/AMS Status Document.
pass
if (tmpStat):
# An error response was received from the source node.
raise Exception(tmpStat.getMessage())
else:
# The file seems not to have been transferred completely.
errMsg = "Size of cloned file wrong (expected: %d/actual: %d)"
raise Exception(errMsg % (fileInfoObj.getFileSize(), fileSize))
logger.debug("Size of cloned Staging File OK: %s", stagFile)
# The file size was correct.
if (checkChecksum):
if ngamsFileUtils.check_checksum(srvObj, fileInfoObj, stagFile):
logger.debug("Checksum of cloned Staging File OK: %s", stagFile)
else:
logger.debug("No Checksum or Checksum Plug-In specified for file")
def _assertEqualsDir(self, dir1, dir2):
# Entries in dir are the same
files1 = os.listdir(dir1)
files2 = os.listdir(dir2)
files1.sort()
files2.sort()
self.assertListEqual(files1, files2)
# Add dir names so we can treat them as proper paths
addDir = lambda d, f: d + os.sep + f
files1 = list(map(addDir, [dir1 for _ in range(len(files1))], files1))
files2 = list(map(addDir, [dir2 for _ in range(len(files2))], files2))
# Files actually have the same size, we recurse on dirs
for idx, f in [(idx, f) for idx, f in enumerate(files1)]:
if os.path.isfile(f):
self.assertEqual(getFileSize(f), getFileSize(files2[idx]))
elif os.path.isdir(f):
self._assertEqualsDir(f, files2[idx])
def __retrieveThread(optDic, taskCtrl, dummy):
"""
"""
info(4, "Entering retrieveThread(%s) ..." % getThreadName())
client = ngamsPClient.ngamsPClient().\
parseSrvList(optDic["servers"][NGAS_OPT_VAL])
while (1):
nextFileId = taskCtrl.getNextFileId()
if (not nextFileId): thread.exit()
nextFileId = nextFileId[:-1]
stat = client.retrieve2File(nextFileId, targetFile=nextFileId)
fileSize = getFileSize(nextFileId)
taskCtrl.incBytesRecv(fileSize)
info(1, "Next File ID: %s" % nextFileId)
def performStaging(srvObj, reqPropsObj, httpRef, filename):
"""
if the staging plugin is set, then perform staging
using the registered staging plugin
if the file is offline (i.e. on Tape)
srvObj: Reference to NG/AMS server class object (ngamsServer).
filename: File to be processed (string).
"""
if srvObj.getCfg().getFileStagingEnable() != 1:
return
fspi = srvObj.getCfg().getFileStagingPlugIn()
if not fspi:
return
logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi,
filename)
isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline')
if isFileOffline(filename) == 0:
return
logger.info("Invoking FSPI.stageFiles: %s to check file: %s", fspi,
filename)
stageFiles = loadPlugInEntryPoint(fspi, 'stageFiles')
try:
st = time.time()
stageFiles(filenames=[filename],
requestObj=reqPropsObj,
serverObj=srvObj)
howlong = time.time() - st
fileSize = getFileSize(filename)
logger.debug('Staging rate = %.0f Bytes/s (%.0f seconds) for file %s',
fileSize / howlong, howlong, filename)
except socket.timeout:
errMsg = 'Staging timed out: %s' % filename
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504)
raise
def bbcp_transfer(request, out_fname, crc_name, skip_crc):
bparam = get_params(request)
# perform the bbcp transfer, we will always return the checksum
start = time.time()
checksum = bbcpFile(request.getFileUri(), out_fname, bparam, crc_name,
skip_crc)
size = getFileSize(out_fname)
totaltime = time.time() - start
# Feedback the file size into the request object now that we know it,
# just in case we need it later
request.setSize(size)
# Artificially split the total time between read and write (+ 0 crc),
# we don't actually know how much was spent on what
half = totaltime / 2
return ngamsArchiveUtils.archiving_results(size, half, half, 0, totaltime,
crc_name, checksum)
def createFitsFiles(testParDic):
"""
Create the test 'archived' files according to the specified parameters.
This function handles FITS files.
Note:
- The files is archived as it is. I.e., if it is not compressed, it
remains like that.
- Even if rearchiving the same file with new File IDs, no parameters of
the header are updated.
- It is assumed that the file has a valid FITS checksum.
- It is assumed that the file has a valid ARCFILE.
- Only new versions are created. I.e., the File ID is incremented at
each iteration.
testParDic: Dictionary with parameters for running the test (dictionary).
Returns: Void.
"""
from ngamsPlugIns import ngamsFitsPlugIn
filesPerDay = 10000 # Max. number of files to generate per day.
baseFile = testParDic["BASEFILE"]
ext = baseFile.split(".")[-1]
if ((ext == "Z") or (ext == "gz")):
try:
tmpFile1 = "/tmp/ngasCreateVolume_%f" % time.time()
tmpFile2 = "%s.%s" % (tmpFile1, ext)
os.system("cp %s %s" % (baseFile, tmpFile2))
os.system("gunzip %s" % tmpFile2)
arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(tmpFile1)
uncomprSize = getFileSize(tmpFile1)
os.system("rm -f %s*" % tmpFile1)
except Exception, e:
os.system("rm -f %s*" % tmpFile1)
raise e
def finfo_from_database(fileInfo, srvObj, reqPropsObj):
fileId = fileInfo.getFileId()
fileVer = fileInfo.getFileVersion()
# Locate the file best suiting the query and send it back if possible.
location, _, ipAddress, port, mountPoint, filename, fileVersion, mimeType = \
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId, fileVersion=fileVer)
basename = os.path.basename(filename)
if location == NGAMS_HOST_LOCAL:
absname = os.path.normpath(os.path.join(mountPoint, filename))
size = getFileSize(absname)
opener = fopener(absname)
elif location == NGAMS_HOST_CLUSTER or location == NGAMS_HOST_REMOTE:
# TODO: int in python2 guarantees at least 32 bits, so we may overflow here
size = int(srvObj.getDb().getFileSize(fileId, fileVersion))
opener = http_opener(ipAddress, port, fileId, fileVersion, srvObj)
else:
raise Exception("Unknown location type: %s" % (location,))
return ngamsMIMEMultipart.file_info(mimeType, basename, size, opener)
1]
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the temp compressed file
cmd2 = 'rm %s' % newfn
execCmd(cmd2)
# move the uncompressed file back
cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename)
ret = execCmd(cmd2)
if (ret[0] != 0):
logger.error('Fail to recover from a failed compression: %s',
cmd2)
return
# change the CRC in the database
new_fs = getFileSize(filename)
query = "UPDATE ngas_files SET checksum = {0}, file_size = {1}, compression = 'RICE', format = 'application/fits' WHERE file_id = {2} AND disk_id = {3} AND file_version = {4}"
logger.debug("Updating CRC SQL: %s", str(query))
if (debug):
pass
else:
try:
srvObj.getDb().query2(query,
args=(crc, new_fs, fileId, diskId,
fileVersion))
except Exception, ex:
errMsg = 'Fail to update crc for file %s/%d/%s: %s' % (
fileId, fileVersion, diskId, str(ex))
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the compressed file that have been copied in
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Compress the file based on file_path (file_id, disk_id, and file_version)
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
required_params = ['file_path', 'file_id', 'file_version', 'disk_id']
parDic = reqPropsObj.getHttpParsDic()
for rp in required_params:
if (not parDic.has_key(rp)):
errMsg = 'Parameter missing: %s' % rp
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
filename = reqPropsObj.getHttpPar('file_path')
logger.debug('Compressing file %s', filename)
if (not os.path.exists(filename)):
errMsg = 'File Not found: %s' % filename
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=404)
return
fileId = parDic['file_id']
fileVersion = int(parDic['file_version'])
diskId = parDic['disk_id']
if (not isMWAVisFile(fileId)):
errMsg = 'Not MWA visibilty file: %' % (fileId)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
if (hasCompressed(filename)):
errMsg = 'OK'
logger.debug('File compressed %s already, return now', filename)
httpRef.send_data(errMsg, NGAMS_TEXT_MT)
return
if (parDic.has_key('scale_factor')):
sf = int(parDic['scale_factor'])
else:
sf = 4
if (parDic.has_key('threshold')):
th = float(parDic['threshold']) # currently not used
if (parDic.has_key('bins')):
bins = int(parDic['bins'])
else:
bins = 0
if (parDic.has_key('remove_uc')):
remove_uc = int(parDic['remove_uc']) # currently not used
if (parDic.has_key('timeout')):
timeout = int(parDic['timeout'])
if (timeout <= 0):
timeout = 600
if (parDic.has_key('debug')):
debug = int(parDic['debug'])
else:
debug = 0
if (bins):
binstr = '-h %d' % bins
else:
binstr = ''
bname = os.path.basename(filename)
newfn = '%s/%s' % (work_dir, bname)
fndir = os.path.dirname(filename)
old_fs = getFileSize(filename)
# do compression
cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn)
logger.debug('Compression: %s', cmd)
if (debug):
pass
else:
ret = execCmd(cmd)
if (ret[0] != 0):
errMsg = 'Failed to compress %s' % ret[1]
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
# calculate the CRC code
logger.debug('Calculating CRC for file %s', newfn)
if (debug):
crc = 1234567
else:
try:
crc = getFileCRC(newfn)
except Exception as exp:
errMsg = 'Failed to calculate the CRC for file %s: %s' % (newfn, str(exp))
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the temp compressed file
cmd1 = 'rm %s' % newfn
execCmd(cmd1)
return
# rename the uncompressed file (under the same directory)
cmd1 = 'mv %s %s/%s_uncompressed' % (filename, fndir, bname)
logger.debug('Renaming the uncompressed file: %s', cmd1)
if (debug):
pass
else:
ret = execCmd(cmd1)
if (ret[0] != 0):
errMsg = 'Failed to move/rename uncompressed file %s: %s' % (filename, ret[1])
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the temp compressed file
cmd1 = 'rm %s' % newfn
execCmd(cmd1)
return
# move the compressed file to replace the uncompressed file
cmd2 = 'mv %s %s' % (newfn, filename)
logger.debug('Moving the compressed file to NGAS volume: %s', cmd2)
if (debug):
pass
else:
ret = execCmd(cmd2)
if (ret[0] != 0):
errMsg = 'Failed to move the compressed file to NGAS volume: %s' % ret[1]
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the temp compressed file
cmd2 = 'rm %s' % newfn
execCmd(cmd2)
# move the uncompressed file back
cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename)
ret = execCmd(cmd2)
if (ret[0] != 0):
logger.error('Fail to recover from a failed compression: %s', cmd2)
return
# change the CRC in the database
new_fs = getFileSize(filename)
query = "UPDATE ngas_files SET checksum = {0}, file_size = {1}, compression = 'RICE', format = 'application/fits' WHERE file_id = {2} AND disk_id = {3} AND file_version = {4}"
logger.debug("Updating CRC SQL: %s", str(query))
if (debug):
pass
else:
try:
srvObj.getDb().query2(query, args=(crc, new_fs, fileId, diskId, fileVersion))
except Exception as ex:
errMsg = 'Fail to update crc for file %s/%d/%s: %s' % (fileId, fileVersion, diskId, str(ex))
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the compressed file that have been copied in
cmd2 = 'rm %s' % filename
execCmd(cmd2)
# recover the uncompressed back
cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename)
ret = execCmd(cmd2)
if (ret[0] != 0):
logger.error('Fail to recover from a failed compression: %s', cmd2)
return
# remove the uncompressed file
errMsg = 'Compression completed %s.' % (filename)
if (not debug):
#new_fs = getFileSize(filename)
cmd3 = 'rm %s/%s_uncompressed' % (fndir, bname)
ret = execCmd(cmd3)
if (ret[0] != 0):
errMsg += '. But fail to remove the uncompressed file %s/%s: %s' % (fndir, bname, ret[1])
logger.warning(errMsg)
logger.debug(errMsg + '\. Result: %d - %d = %d, compress ratio: %.2f' % (old_fs, new_fs, (old_fs - new_fs), new_fs * 1.0 / old_fs))
else:
logger.debug(errMsg)
httpRef.send_data(errMsg + '\n', NGAMS_TEXT_MT)
def checkCorrectFileSizeDb(diskId, correct=0, notifEmail=None):
"""
Check the size of the files registered on the given disk. If a file
has a size different from the one stored in the DB, the size is
corrected in the DB
diskId: ID of disk for which to check the files (string).
Returns: Void
"""
server, db, user, password = ngasUtilsLib.getDbPars()
db = ngamsDb.ngamsDb(server, db, user, password, createSnapshot=0)
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().read(getHostName(), db, diskId)
if (tmpDiskInfo.getHostId() != getHostName()):
raise Exception, "The disk: %s is not inserted in this system!!" %\
diskId
notifMsg = "\nFILE SIZE CHECK/CORRECTION REPORT:\n" +\
"(only files with an incorrect size in the DB are reported)\n"+\
"\nDisk ID: " + diskId + "\n" +\
128 * "="
sys.stdout.write(notifMsg)
dbCursor = db.getFileSummary1(getHostName(), [diskId], fileStatus=[])
while (1):
res = dbCursor.fetch(128)
if (not res): break
for fileInfo in res:
# IMPL: This works only on calibration frames for the moment.
# (these are uncompressed -> file size == uncompr file size)
filename = fileInfo[2]
if (filename.find("/M.") == -1): continue
mountPoint = fileInfo[1]
dbChecksum = fileInfo[3]
fileId = fileInfo[5]
fileVer = fileInfo[6]
dbFileSize = int(fileInfo[7])
diskId = fileInfo[9]
complFilename = os.path.normpath(mountPoint + "/" + filename)
diskFileSize = getFileSize(complFilename)
msg = "\nCheck size of file: %s/%s/%d: " % (diskId, fileId,
fileVer)
if (diskFileSize == dbFileSize):
msg += "Size OK"
else:
msg += "SIZE WRONG!"
# Check checksum.
from ngamsPlugIns import ngamsGenCrc32
checksum = ngamsGenCrc32.ngamsGenCrc32(None, complFilename, 0)
if (checksum != dbChecksum):
msg += " ILLEGAL CHECKSUM!"
else:
msg += " CHECKSUM OK!"
if (correct):
tmpFileInfo = ngamsFileInfo.\
ngamsFileInfo().read(db, fileId, fileVer,
diskId)
tmpFileInfo.setFileSize(diskFileSize)
tmpFileInfo.setUncompressedFileSize(diskFileSize)
tmpFileInfo.setFileStatus(NGAMS_FILE_STATUS_OK)
if (not tmpFileInfo.getCreationDate()):
tmpFileInfo.setCreationDate(tmpFileInfo.\
getIngestionDate())
tmpFileInfo.write(getHostName(), db, 0, 1)
msg += " FILE SIZE CORRECTED!"
notifMsg += msg
sys.stdout.write(msg)
sys.stdout.flush()
notifMsg += "\n%s\n" % (128 * "=")
print "\n" + 128 * "="
if (notifEmail):
ngasUtilsLib.sendEmail("FILE SIZE CHECK/CORRECTION REPORT", notifEmail,
notifMsg, "text/plain",
"FILE-SIZE-CHECK-REP-%s" % diskId)
def _httpPostUrl(url,
mimeType,
contDisp = "",
dataRef = "",
dataSource = "BUFFER",
dataTargFile = "",
blockSize = 65536,
suspTime = 0.0,
timeOut = None,
authHdrVal = "",
dataSize = -1,
session_uuid = ""):
"""
Post the the data referenced on the given URL. This function is adapted from
ngamsLib.httpPostUrl, which does not support block-level suspension and cancelling for file transfer
The data send back from the remote server + the HTTP header information
is return in a list with the following contents:
[<HTTP status code>, <HTTP status msg>, <HTTP headers (list)>, <data>]
url: URL to where data is posted (string).
mimeType: Mime-type of message (string).
contDisp: Content-Disposition of the data (string).
dataRef: Data to post or name of file containing data to send
(string).
dataSource: Source where to pick up the data (string/BUFFER|FILE|FD).
dataTargFile: If a filename is specified with this parameter, the
data received is stored into a file of that name (string).
blockSize: Block size (in bytes) used when sending the data (integer).
suspTime: Time in seconds to suspend between each block (double).
timeOut: Timeout in seconds to wait for replies from the server
(double).
authHdrVal: Authorization HTTP header value as it should be sent in
the query (string).
dataSize: Size of data to send if read from a socket (integer).
Returns: List with information from reply from contacted
NG/AMS Server (reply, msg, hdrs, data) (list).
"""
T = TRACE()
# Separate the URL from the command.
idx = (url[7:].find("/") + 7)
tmpUrl = url[7:idx]
cmd = url[(idx + 1):]
http = httplib.HTTP(tmpUrl)
logger.debug("HTTP Header: %s: %s", NGAMS_HTTP_POST, cmd)
http.putrequest(NGAMS_HTTP_POST, cmd)
logger.debug("HTTP Header: Content-Type: %s", mimeType)
http.putheader("Content-Type", mimeType)
if (contDisp != ""):
logger.debug("HTTP Header: Content-Disposition: %s", contDisp)
http.putheader("Content-Disposition", contDisp)
if (authHdrVal):
if (authHdrVal[-1] == "\n"): authHdrVal = authHdrVal[:-1]
logger.debug("HTTP Header: Authorization: %s", authHdrVal)
http.putheader("Authorization", authHdrVal)
if (dataSource == "FILE"):
dataSize = getFileSize(dataRef)
elif (dataSource == "BUFFER"):
dataSize = len(dataRef)
if (dataSize != -1):
logger.debug("HTTP Header: Content-Length: %s", str(dataSize))
http.putheader("Content-Length", str(dataSize))
logger.debug("HTTP Header: Host: %s", getHostName())
http.putheader("Host", getHostName())
http.endheaders()
logger.debug("HTTP header sent")
http._conn.sock.settimeout(timeOut)
# Send the data.
logger.debug("Sending data ...")
if (dataSource == "FILE"):
fdIn = open(dataRef)
block = "-"
blockAccu = 0
http._conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 87380) # to fit Fornax
while (block != ""):
"""
if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0):
info(3, "Received cancel/suspend request, discard remaining blocks")
break
"""
block = fdIn.read(blockSize)
blockAccu += len(block)
http._conn.sock.sendall(block)
if (suspTime > 0.0): time.sleep(suspTime)
fdIn.close()
elif (dataSource == "FD"):
fdIn = dataRef
dataRead = 0
while (dataRead < dataSize):
if ((dataSize - dataRead) < blockSize):
rdSize = (dataSize - dataRead)
else:
rdSize = blockSize
block = fdIn.read(rdSize)
http._conn.sock.sendall(block)
dataRead += len(block)
if (suspTime > 0.0): time.sleep(suspTime)
else:
# dataSource == "BUFFER"
http.send(dataRef)
logger.debug("Data sent")
if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0):
logger.debug("Received cancel/suspend request, close HTTP connection and return None values")
if (http != None):
http.close()
del http
return [None, None, None, None]
# Receive + unpack reply.
logger.debug("Waiting for reply ...")
reply, msg, hdrs = http.getreply()
if (hdrs == None):
errMsg = "Illegal/no response to HTTP request encountered!"
raise Exception(errMsg)
if (hdrs.has_key("content-length")):
dataSize = int(hdrs["content-length"])
else:
dataSize = 0
if (dataTargFile == ""):
data = http.getfile().read(dataSize)
else:
fd = None
try:
data = dataTargFile
fd = open(dataTargFile, "w")
fd.write(http.getfile().read(dataSize))
fd.close()
except Exception as e:
if (fd != None): fd.close()
raise e
# Dump HTTP headers if Verbose Level >= 4.
logger.debug("HTTP Header: HTTP/1.0 " + str(reply) + " " + msg)
if logger.isEnabledFor(logging.DEBUG):
for hdr in hdrs.keys():
logger.debug("HTTP Header: %s: %s", hdr, hdrs[hdr])
if (http != None):
http.close()
del http
return [reply, msg, hdrs, data]
def checkFile(srvObj,
sum1FileInfo,
checkReport,
skipCheckSum = 0,
executor=None):
"""
Function to carry out a consistency check on a file.
If `stop_evt` and `allowed_evt` are given, then `get_checksum_interruptible`
is used internally by this method; otherwise `get_checksum` is used.
If `executor` is given, then it is used to carry out the execution of the
checksum calculation; otherwise `get_checksum` is used.
"""
executor = executor or get_checksum
foundProblem = 0
fileInfo = sum1FileInfo
diskId = fileInfo[ngamsDbCore.SUM1_DISK_ID]
slotId = fileInfo[ngamsDbCore.SUM1_SLOT_ID]
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] + "/" +\
fileInfo[ngamsDbCore.SUM1_FILENAME])
if (fileInfo[ngamsDbCore.SUM1_CHECKSUM] == None):
checksumDb = ""
else:
checksumDb = fileInfo[ngamsDbCore.SUM1_CHECKSUM].strip()
crc_variant = fileInfo[ngamsDbCore.SUM1_CHECKSUM_PI]
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = fileInfo[ngamsDbCore.SUM1_VERSION]
dbFileSize = fileInfo[ngamsDbCore.SUM1_FILE_SIZE]
logger.debug("Checking file with ID: %s and filename: %s on NGAS host: %s",
fileId, filename, srvObj.getHostId())
## Set the file status "checking bit" temporary to 1.
#tmpFileStatus = fileStatus[0] + "1" + fileStatus[2:]
#srvObj.getDb().setFileStatus(fileId, fileVersion, diskId, tmpFileStatus)
# Create name of temporary file in "<NGAS Rt Pt>/cache" indicating which
# file is being checked:
# <NGAS Rt Pt>/cache/<Thr ID>___<Disk ID>___<File ID>___<File Version>.\
# check
fileChecked = os.path.normpath("%s|%s___%s___%s___%s.check" %
(
NGAMS_CACHE_DIR, NGAMS_DATA_CHECK_THR,
diskId, fileId,
str(fileVersion))).replace("/", "_")
fileChecked = srvObj.getCfg().getRootDirectory() + '/' + fileChecked.replace("|","/")
fileCheckedFo = None
try:
# Create file indicating which data file is being checked.
rmFile(fileChecked)
fileCheckedFo = open(fileChecked, "w")
fileCheckedFo.close()
# Check if file exists.
fileExists = os.path.exists(filename)
if (not fileExists):
foundProblem = 1
logger.error('File %s does not exist on disk', filename)
checkReport.append(["ERROR: File in DB missing on disk", fileId,
fileVersion, slotId, diskId, filename])
srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False)
# Check if file has the correct size.
if ((not foundProblem) and fileExists):
fileSize = getFileSize(filename)
if (fileSize != dbFileSize):
foundProblem = 1
logger.error('File %s has wrong size. Expected: %d/Actual: %d',
filename, dbFileSize, fileSize)
format = "ERROR: File has wrong size. Expected: %d/Actual: %d."
checkReport.append([format % (dbFileSize , fileSize), fileId,
fileVersion, slotId, diskId, filename])
srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False)
# Check checksum if this test should not be skipped.
if ((not foundProblem) and (not skipCheckSum)):
if crc_variant is None:
crc_variant = srvObj.getCfg().getCRCVariant()
checksum_info = get_checksum_info(crc_variant)
if crc_variant is not None:
try:
blockSize = srvObj.getCfg().getBlockSize()
if blockSize == -1:
blockSize = 4096
checksum_typ = get_checksum_name(crc_variant)
# Calculate the checksum, possibly under the executor
start = time.time()
checksumFile = executor(blockSize, filename, crc_variant)
duration = time.time() - start
fsize_mb = getFileSize(filename) / 1024. / 1024.
logger.info("Checked %s in %.4f [s] using %s. Check ran at %.3f [MB/s]. Checksum file/db: %s / %s",
filename, duration, str(crc_variant), fsize_mb / duration,
str(checksumFile), checksumDb)
except Exception:
# We assume an IO error:
# "[Errno 2] No such file or directory"
# This problem has already been registered further
# up so we ignore it here.
rmFile(fileChecked)
logger.exception("Error while checking file %s", filename)
return
else:
checksumFile = ""
if not checksumDb and checksumFile:
checkReport.append(["NOTICE: Missing checksum - generated",
fileId, fileVersion, slotId, diskId,
filename])
srvObj.getDb().setFileChecksum(srvObj.getHostId(),
fileId, fileVersion, diskId,
str(checksumFile), checksum_typ)
elif not checksumDb and not checksumFile:
checkReport.append(["NOTICE: Missing checksum - " +\
"cannot generate - update DB",
fileId, fileVersion, slotId, diskId,
filename])
elif not checksum_info.equals(checksumDb, checksumFile):
logger.error("File %s has inconsistent checksum! file/db: %s / %s",
filename, str(checksumFile), checksumDb)
checkReport.append(["ERROR: Inconsistent checksum found",
fileId, fileVersion, slotId, diskId,
filename])
srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False)
# If file is OK but is marked as 'bad', reset the flag.
if not foundProblem:
srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, True)
## Set the checking flag back to 0 in the DB.
#srvObj.getDb().setFileStatus(fileId, fileVersion, diskId, fileStatus)
# Reset file indicating which data file is being checked.
rmFile(fileChecked)
except Exception:
if (fileCheckedFo): fileCheckedFo.close()
# Reset file indicating which data file is being checked.
rmFile(fileChecked)
raise
def _filesSize(self):
return sum([getFileSize(f) for f in self.myfiles])
ext = baseFile.split(".")[-1]
if ((ext == "Z") or (ext == "gz")):
try:
tmpFile1 = "/tmp/ngasCreateVolume_%f" % time.time()
tmpFile2 = "%s.%s" % (tmpFile1, ext)
os.system("cp %s %s" % (baseFile, tmpFile2))
os.system("gunzip %s" % tmpFile2)
arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(tmpFile1)
uncomprSize = getFileSize(tmpFile1)
os.system("rm -f %s*" % tmpFile1)
except Exception, e:
os.system("rm -f %s*" % tmpFile1)
raise e
else:
arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(baseFile)
uncomprSize = getFileSize(baseFile)
insId = arcFile.split(".")[0]
isoTime = arcFile[(arcFile.find(".") + 1):]
baseTimeMjd = PccUtTime.TimeStamp(isoTime).getMjd()
# Generate 'static' file info for this run.
ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" %\
testParDic["MOUNTPOINT"])
ngasDiskInfoDoc = open(ngasDiskInfo).read()
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackXmlDoc(ngasDiskInfoDoc, ignoreVarDiskPars=1)
diskId = diskInfoObj.getDiskId()
fileVer = 1
if (ext == "Z"):
format = "application/x-cfits"
compression = "compress -f"
def test_AppendRemove(self):
# Server and client
self.prepExtSrv()
# Create a container, shouldn't be a problem
containerName = "testing"
self.ccreate(containerName)
self._checkContainerClosed(containerName, False)
#------------------------------------------------------------------
# We start testing with a single file append/removal,
# which is a simple use case
#------------------------------------------------------------------
# Archive a single file, append it to the container
# and check the file is there and it contributes to the container size
myfile = self.myfiles[0]
myfileId = os.path.basename(myfile)
myfileSize = getFileSize(myfile)
self.qarchive(myfile, "application/octet-stream")
self.cappend(myfileId, containerName=containerName)
self._checkFilesAndContainerSize(containerName, 1, myfileSize)
self._checkContainerClosed(containerName, False)
# Remove the file now and check that the file is not there anymore,
# decreasing the container size
self.cremove(myfileId, containerName=containerName)
self._checkFilesAndContainerSize(containerName, 0, 0)
self._checkContainerClosed(containerName, False)
#------------------------------------------------------------------
# We continue testing with a list of files for append/removal,
# which is a bit more complex
#------------------------------------------------------------------
# Store the rest of the files
# Append all files to the container (including the first one, we removed it)
# and check that files are there, and that they contribute to the container size
fileIds = ':'.join([os.path.basename(f) for f in self.myfiles])
allFilesSize = self._filesSize()
for myfile in self.myfiles[1:]:
self.qarchive(myfile, "application/octet-stream")
self.cappend(None, fileIdList=fileIds, containerName=containerName)
self._checkFilesAndContainerSize(containerName, len(self.myfiles),
allFilesSize)
self._checkContainerClosed(containerName, False)
# Remove the files now and check that the files are not part of the container anymore,
# decreasing the container size
self.cremove(None, fileIdList=fileIds, containerName=containerName)
self._checkFilesAndContainerSize(containerName, 0, 0)
self._checkContainerClosed(containerName, False)
#------------------------------------------------------------------
# Now we re-archive the same files, effectively creating a second
# version of them. We then check which version is considered as
# part of the container.
#
# This test comes in two flavors:
# - Add the files to the container, re-archive them
# - Re-archive the files, then add them to the container
#------------------------------------------------------------------
# Add the files to the container and then re-archive them
# The new version of the files should be 2, and the new container size
# should correspond to the new files' sizes
self._createFiles()
allFilesSize = self._filesSize()
self.cappend(None, fileIdList=fileIds, containerName=containerName)
for myfile in self.myfiles:
self.qarchive(myfile, "application/octet-stream")
self._checkFilesAndContainerSize(containerName,
len(self.myfiles),
allFilesSize,
fileVersion=2)
self._checkContainerClosed(containerName, False)
# Remove all files and check that the container is empty
self.cremove(None, fileIdList=fileIds, containerName=containerName)
self._checkFilesAndContainerSize(containerName, 0, 0)
self._checkContainerClosed(containerName, False)
# Re-archive the files, add them to the container and check again
# The new version of the files should be 3, and the new container size
# should correspond to the new files' sizes
self._createFiles()
allFilesSize = self._filesSize()
for myfile in self.myfiles:
self.qarchive(myfile, "application/octet-stream")
self.cappend(None, fileIdList=fileIds, containerName=containerName)
self._checkFilesAndContainerSize(containerName,
len(self.myfiles),
allFilesSize,
fileVersion=3)
self._checkContainerClosed(containerName, False)
# Remove all files and check that the container is empty
self.cremove(None, fileIdList=fileIds, containerName=containerName)
self._checkFilesAndContainerSize(containerName, 0, 0)
self._checkContainerClosed(containerName, False)
#------------------------------------------------------------------
# The final test in this section is about "closing" a container
# When a container is created via CCREATE it remains "opened",
# until a CAPPEND command specifies that it wants to "close" the
# container. A container is closed when its ingestion date is set.
#------------------------------------------------------------------
# First of all, check that the container is still currently opened
self._checkContainerClosed(containerName, False)
# Append a file, check that the container is still opened
myfileId = os.path.basename(self.myfiles[0])
self.cappend(myfileId, containerName=containerName)
self._checkContainerClosed(containerName, False)
# Append a second file and mark the container as closed
myfileId = os.path.basename(self.myfiles[1])
self.cappend(myfileId,
containerName=containerName,
closeContainer=True)
self._checkContainerClosed(containerName, True)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Compress the file based on file_path (file_id, disk_id, and file_version)
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
required_params = ['file_path', 'file_id', 'file_version', 'disk_id']
parDic = reqPropsObj.getHttpParsDic()
for rp in required_params:
if (not parDic.has_key(rp)):
errMsg = 'Parameter missing: %s' % rp
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
filename = reqPropsObj.getHttpPar('file_path')
logger.debug('Compressing file %s', filename)
if (not os.path.exists(filename)):
errMsg = 'File Not found: %s' % filename
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=404)
return
fileId = parDic['file_id']
fileVersion = int(parDic['file_version'])
diskId = parDic['disk_id']
if (not isMWAVisFile(fileId)):
errMsg = 'Not MWA visibilty file: %' % (fileId)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
if (hasCompressed(filename)):
errMsg = 'OK'
logger.debug('File compressed %s already, return now', filename)
httpRef.send_data(errMsg, NGAMS_TEXT_MT)
return
if (parDic.has_key('scale_factor')):
sf = int(parDic['scale_factor'])
else:
sf = 4
if (parDic.has_key('threshold')):
th = float(parDic['threshold']) # currently not used
if (parDic.has_key('bins')):
bins = int(parDic['bins'])
else:
bins = 0
if (parDic.has_key('remove_uc')):
remove_uc = int(parDic['remove_uc']) # currently not used
if (parDic.has_key('timeout')):
timeout = int(parDic['timeout'])
if (timeout <= 0):
timeout = 600
if (parDic.has_key('debug')):
debug = int(parDic['debug'])
else:
debug = 0
if (bins):
binstr = '-h %d' % bins
else:
binstr = ''
bname = os.path.basename(filename)
newfn = '%s/%s' % (work_dir, bname)
fndir = os.path.dirname(filename)
old_fs = getFileSize(filename)
# do compression
cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn)
logger.debug('Compression: %s', cmd)
if (debug):
pass
else:
ret = execCmd(cmd)
if (ret[0] != 0):
errMsg = 'Failed to compress %s' % ret[1]
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
return
# calculate the CRC code
logger.debug('Calculating CRC for file %s', newfn)
if (debug):
crc = 1234567
else:
try:
crc = getFileCRC(newfn)
except Exception, exp:
errMsg = 'Failed to calculate the CRC for file %s: %s' % (newfn,
str(exp))
logger.error(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500)
# remove the temp compressed file
cmd1 = 'rm %s' % newfn
execCmd(cmd1)
return
