def copy_to_ngas(self, file_suffix=''):
src_file = "src/SmallFile.fits"
tmp_src_file = self.ngas_path(
"FitsStorage2-Main-3/saf/test/SmallFile.fits" + file_suffix)
checkCreatePath(os.path.dirname(tmp_src_file))
self.cp(src_file, tmp_src_file)
return tmp_src_file
def bbcp_transfer(request, out_fname, crc_name, skip_crc):
# source_file must have the IP of the client, so bbcp can contact it and
# launch a copy ot itself.
# OTOH, target_file must have the IP through which we received the original
# HTTP request to ensure the same network path is used by bbcp
bparam = get_params(request)
source_file = get_source_file(request)
target_file = request.getHttpHdr('host') + ':' + out_fname
# Make target file writable if existing.
if (os.path.exists(out_fname)):
os.chmod(out_fname, 420)
checkCreatePath(os.path.dirname(out_fname))
# perform the bbcp transfer, we will always return the checksum
start = time.time()
checksum = bbcpFile(source_file, target_file, bparam, crc_name, skip_crc)
size = getFileSize(out_fname)
totaltime = time.time() - start
# Feedback the file size into the request object now that we know it,
# just in case we need it later
request.setSize(size)
# Artificially split the total time between read and write (+ 0 crc),
# we don't actually know how much was spent on what
half = totaltime / 2
return ngamsArchiveUtils.archiving_results(size, half, half, 0, totaltime,
crc_name, checksum)
def test_DbSnapshot_6(self):
"""
Synopsis:
Test that DB Snapshot is updated correctly when file are registered.
Description:
The purpose of the test is to verify that the DB Snapshot of a disk
on which files are being registered, is properly updated according
to the files registered.
Expected Result:
After a short while, the information for the files registered should
appear in the DB Snapshot of the disk.
Test Steps:
- Start server.
- Copy over a small test FITS file in 3 copies.
- Send a REGISTER Command to register the files copied over
(specifying root directory of the files).
- Check that the DB Snapshot is updated within a given period of time.
Remarks:
...
"""
self._prepSrv()
regTestDir = self.ngas_path("FitsStorage1-Main-1/reg_test")
checkCreatePath(regTestDir)
for n in range(3):
self.cp('src/SmallFile.fits',
os.path.join(regTestDir, "%d.fits" % n))
self.get_status(NGAMS_REGISTER_CMD, pars=[["path", regTestDir]])
_checkContDbSnapshot(self, 6, ["FitsStorage1-Main-1"], 1, 1)
def copy_and_register(self, file_suffix='', **extra_params):
srcFile = "src/SmallFile.fits"
tmpSrcFile = self.ngas_path(
"FitsStorage2-Main-3/saf/test/SmallFile.fits" + file_suffix)
checkCreatePath(os.path.dirname(tmpSrcFile))
self.cp(srcFile, tmpSrcFile)
return self.get_status(NGAMS_REGISTER_CMD, (("path", tmpSrcFile), ) +
tuple(extra_params.items()))
def test_unregistered(self):
# Manually copy a file into the disk
trgFile = tmp_path("NGAS/FitsStorage1-Main-1/SmallFile.fits")
checkCreatePath(os.path.dirname(trgFile))
self.cp("src/SmallFile.fits", trgFile)
# It should appear as unregistered when the server checks it
cfg, db = self.start_srv(delDirs=False)
self.wait_and_count_checked_files(cfg, db, 0, 1, 0)
def test_unregistered(self):
# Manually copy a file into the disk
checkCreatePath('/tmp/ngamsTest/NGAS/FitsStorage1-Main-1/')
trgFile = "/tmp/ngamsTest/NGAS/FitsStorage1-Main-1/SmallFile.fits"
shutil.copy("src/SmallFile.fits", trgFile)
# It should appear as unregistered when the server checks it
cfg, db = self.start_srv(delDirs=False)
self.wait_and_count_checked_files(cfg, db, 0, 1, 0)
def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = tmp_path("NGAS")
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(
['tar', 'zxf',
self.resource('src/volumes_dir.tar.gz')])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
self.prepExtSrv(delDirs=0, cfgFile='src/ngamsCfg_VolumeDirectory.xml')
# Archive a file.
stat = self.archive("src/SmallFile.fits")
self.assert_status_ref_file(
"ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref", stat)
# Check that the target files have been archived in their
# appropriate locations.
trgFile = tmp_path("test_VolumeDir_01_tmp")
refFile = "src/SmallFile.fits"
outFilePath = tmp_path("SmallFile.fits")
stat = self.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(refFile, outFilePath, "Retrieved file incorrect")
def test_RegisterCmd_1(self):
"""
Synopsis:
REGISTER Command/register single file compl. path.
Description:
Test handling of the REGISTER Command under normal circumstances.
Expected Result:
The REGISTER Command should be accepted by the server and should
be executed successfully.
Test Steps:
- Start server.
- Copy file onto NGAS Disk.
- Submit REGISTER Command requesting to register the file copied over
(wait=1).
- Check response from the server that the request was successfully
executed.
- Check the DB info for the registered file.
Remarks:
...
"""
_, dbObj = self.prepExtSrv()
srcFile = "src/SmallFile.fits"
tmpSrcFile = "/tmp/ngamsTest/NGAS/" +\
"FitsStorage2-Main-3/saf/test/SmallFile.fits"
checkCreatePath(os.path.dirname(tmpSrcFile))
shutil.copy(srcFile, tmpSrcFile)
tmpStatFile = sendExtCmd(8888, NGAMS_REGISTER_CMD,
[["path", tmpSrcFile]])
refStatFile = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref"
self.checkFilesEq(refStatFile, tmpStatFile,
"Incorrect status returned for REGISTER command")
diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3"
filePrefix = "ngamsRegisterCmdTest_test_RegisterCmd_1"
fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref"
fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp"
fileId = "TEST.2001-05-08T15:25:00.123"
startTime = time.time()
host_id = getHostName() + ":8888"
while ((time.time() - startTime) < 10):
tmpFileRes = dbObj.getFileInfoFromFileIdHostId(
host_id, fileId, 1, diskId)
if (tmpFileRes): break
if not tmpFileRes:
self.fail(
"Couldn't get fileInfo result from database within 10 seconds")
tmpFileObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(tmpFileRes)
saveInFile(fileInfoTmp, filterDbStatus1(tmpFileObj.dumpBuf()))
self.checkFilesEq(fileInfoRef, fileInfoTmp,
"Incorrect info in DB for registered file")
def _openDbSnapshot(ngamsCfgObj,
mtPt):
"""
Open a bsddb file DB. If the file exists and this is not
a read-only NGAS system the file is opened for reading and writing.
If this is a read-only NGAS system it is only opened for reading.
If the file DB does not exist, a new DB is created.
If the file DB does not exist and this is a read-only NGAS system,
None is returned.
The name of the DB file is:
<Disk Mount Point>/NGAMS_DB_DIR/NGAMS_DB_NGAS_FILES
ngamsCfgObj: NG/AMS Configuration Object (ngamsConfig).
mtPt: Mount point (string).
Returns: File DB object (bsddb|None).
"""
snapShotFile = os.path.normpath(mtPt + "/" + NGAMS_DB_DIR + "/" +\
NGAMS_DB_NGAS_FILES)
checkCreatePath(os.path.normpath(mtPt + "/" + NGAMS_DB_CH_CACHE))
if (os.path.exists(snapShotFile)):
if (_updateSnapshot(ngamsCfgObj)):
# Open the existing DB Snapshot for reading and writing.
snapshotDbm = bsddb.hashopen(snapShotFile, "w")
else:
# Open only for reading.
snapshotDbm = bsddb.hashopen(snapShotFile, "r")
else:
if (_updateSnapshot(ngamsCfgObj)):
# Create a new DB Snapshot.
snapshotDbm = bsddb.hashopen(snapShotFile, "c")
else:
# There is no DB Snapshot and it is not possible to
# create one - the check cannot be carried out.
snapshotDbm = None
# Remove possible, old /<mt pt>/.db/NgasFiles.xml snapshots.
# TODO: Remove when it can be assumed that all old XML snapshots have
# been removed.
rmFile(os.path.normpath(mtPt + "/" + NGAMS_DB_DIR + "/NgasFiles.xml"))
return snapshotDbm
def checkSystem(testParDic):
"""
Carry out a system check for the given system
testParDic: Dictionary with parameters for running the test (dictionary).
Returns: Void.
"""
info(4, "Entering serverLoop() ...")
checkCreatePath(testParDic["WORKING-DIR"])
srvList = testParDic["SERVERS"].split(",")
srvList.sort()
if (testParDic["IGNORE-SERVERS"]):
ignoreSrvList = testParDic["IGNORE-SERVERS"].split(",")
ignoreSrvList.sort()
else:
ignoreSrvList = []
domainList = testParDic["DOMAINS"].split(",")
domainList.sort()
dbSrv, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(dbSrv, db, user, password, 0)
srvStr = str(srvList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
domStr = str(domainList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
dbmName = os.path.normpath("%s/NGAS_SYS_CHECK_%s_%s_%s.bsddb" %\
(testParDic["WORKING-DIR"], os.environ["USER"],
srvStr, domStr))
dbmObj = ngamsDbm.ngamsDbm(dbmName, writePerm=1)
testParDic["_testTmpPath"] = os.path.normpath("%s/NGAS_SYS_CHECK_%.6f" %\
(testParDic["WORKING-DIR"],
time.time()))
os.system("mkdir -p %s" % testParDic["_testTmpPath"])
try:
serverLoop(testParDic, srvList, ignoreSrvList, domainList, dbCon,
dbmObj)
testParDic["_endTime"] = time.time()
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
except Exception, e:
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
raise Exception, e
def test_NormalExec_2(self):
"""
Synopsis:
File existing, remove via path, +/-execute.
Description:
The purpose of the test is to verify the normal functioning of the
DISCARD Command when DISCARD'ing files referred to by their path.
It is tried both to execute the command with execute=0 and execute=1.
The file is not registered in the NGAS DB.
This simulates the situation where spurious files are removed from the
NGAS system only referred to by their path.
Expected Result:
When the DISCARD Command is issued, execute=0, the NG/AMS Server should
find the file and return a response indicating that the file is only
available in 1 copy. When submitting the DISCARD Command with
execute=1, the file should disappear from the disk.
Test Steps:
- Start NG/AMS Server.
- Copy a test file onto a disk of the NGAS System.
- Issue a DISCARD Command specifying the copied file (execute=0).
- Check that the response indicates that the DISCARD Request is
granted.
- Reissue the DISCARD Command specifying the copied file (execute=1).
- Check that the response indicates that the file has been removed.
- Check that the file has disappeared from the disk.
Remarks:
TODO!: Implement check to verify that file has been removed from the
disk.
"""
self.prepExtSrv()
trgFile = self.ngas_path("FitsStorage3-Main-5/saf/SmallFile.fits")
checkCreatePath(os.path.dirname(trgFile))
self.cp('src/SmallFile.fits', trgFile)
pars = [["path", trgFile]]
ref_file = "ref/ngamsDiscardCmdTest_test_NormalExec_2_%d_ref"
self._assert_discard(ref_file, pars)
def _test_archive_retrieve_in_cluster(self, as_tar):
self.prepCluster((8888, 8889))
container_name = 'toplevel'
# Create our own new "root" to easily use self._assertEqualsDir later
src_root = tmp_path('src', container_name)
checkCreatePath(src_root)
cp = lambda x: self.cp(x, os.path.join(src_root, os.path.basename(x)))
cp(self.myfiles[0])
cp(self.myfiles[1])
# Create a container and archive two files into it with our clients
def archive_into_container(port, fname):
self.archive(port, fname, "application/octet-stream")
self.cappend(port,
os.path.basename(fname),
containerName=container_name)
self.ccreate(8888, container_name)
archive_into_container(8888, self.myfiles[0])
archive_into_container(8889, self.myfiles[1])
# CRETRIEVE the container using both clients in turns,
# both files should come back
def retrieve_container(port):
tgt_dir = tmp_path('tgt')
self.cretrieve(port,
container_name,
targetDir=tgt_dir,
as_tar=as_tar)
self._assertEqualsDir(src_root, os.path.join(tgt_dir, 'toplevel'))
rmFile(tgt_dir)
retrieve_container(8888)
retrieve_container(8889)
def saveToFile(srvObj,
ngamsCfgObj,
reqPropsObj,
trgFilename,
blockSize,
startByte):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
disk_id = reqPropsObj.fileinfo['diskId']
source_host = reqPropsObj.fileinfo['sourceHost']
host_id = reqPropsObj.fileinfo['hostId']
file_version = reqPropsObj.fileinfo['fileVersion']
file_id = reqPropsObj.fileinfo['fileId']
host, port = source_host.split(":")
pars = {
'disk_id': disk_id,
'host_id': host_id,
'quick_location': '1',
'file_version': file_version,
'file_id': file_id
}
hdrs = {'Range': str(startByte) + '-'}
rx_timeout = 30 * 60
if srvObj.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(srvObj.getCfg().getVal("Mirroring[1].rx_timeout"))
response = ngamsHttpUtils.httpGet(host, int(port), 'RETRIEVE', pars=pars, hdrs=hdrs, timeout=rx_timeout)
# can we resume a previous download?
downloadResumeSupported = 'bytes' in response.getheader("Accept-Ranges", '')
logger.debug("Creating path: %s", trgFilename)
checkCreatePath(os.path.dirname(trgFilename))
crc_info = ngamsFileUtils.get_checksum_info('crc32')
if startByte != 0:
logger.info("resume requested")
if startByte != 0 and downloadResumeSupported:
logger.info("Resume requested and mirroring source supports resume. Appending data to previously started staging file")
crc = ngamsFileUtils.get_checksum(65536, trgFilename, 'crc32')
reqPropsObj.setBytesReceived(startByte)
fdOut = open(trgFilename, "a")
else:
if (startByte > 0):
logger.info("Resume of download requested but server does not support it. Starting from byte 0 again.")
fdOut = open(trgFilename, "w")
crc = crc_info.init
start = time.time()
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
logger.info("It is an HTTP Archive Pull Request: trying to get Content-length")
hdrs = {h[0]: h[1] for h in response.getheaders()}
if hdrs.has_key('content-length'):
remSize = int(hdrs['content-length'])
else:
logger.warning("Non Content-Lenght header found, defaulting to 1e11")
remSize = int(1e11)
# Receive the data.
buf = "-"
rdSize = blockSize
crc_m = crc_info.method
with contextlib.closing(response), contextlib.closing(fdOut):
while (remSize > 0):
if (remSize < rdSize):
rdSize = remSize
buf = response.read(rdSize)
sizeRead = len(buf)
if sizeRead == 0:
raise Exception("server is unreachable")
else:
crc = crc_m(buf, crc)
remSize -= sizeRead
reqPropsObj.setBytesReceived(reqPropsObj.getBytesReceived() +\
sizeRead)
fdOut.write(buf)
crc = crc_info.final(crc)
deltaTime = time.time() - start
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. " +\
"Rate: %.2f Bytes/s"
logger.info(msg, trgFilename, int(reqPropsObj.getBytesReceived()),
deltaTime, (float(reqPropsObj.getBytesReceived()) /
deltaTime))
# Raise exception if less byes were received as expected.
if (remSize != 0):
msg = "No all expected data arrived, %d bytes left to read" % (remSize,)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
# now check the CRC value against what we expected
sourceChecksum = reqPropsObj.checksum
logger.info('source checksum: %s - current checksum: %d', str(sourceChecksum), crc)
if (crc != int(sourceChecksum)):
msg = "checksum mismatch: source=" + str(sourceChecksum) + ", received: " + str(crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
return [deltaTime,crc]
def save_to_file(ngams_server, request_properties, target_filename):
"""
Save the data available on an HTTP channel into the given file
:param ngams_server: NG/AMS Configuration object (ngamsConfig)
:param request_properties: NG/AMS Request Properties object (ngamsReqProps)
:param target_filename: Target name for file where data will be written (string)
:param block_size: Block size (bytes) to apply when reading the data from the HTTP channel (integer)
:param start_byte: Start byte offset
:return: Tuple. Element 0: Time in took to write file (s) (tuple)
"""
source_host = request_properties.fileinfo['sourceHost']
file_version = request_properties.fileinfo['fileVersion']
file_id = request_properties.fileinfo['fileId']
logger.info("Creating path: %s", target_filename)
checkCreatePath(os.path.dirname(target_filename))
rx_timeout = 30 * 60
if ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(
ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"))
host, port = source_host.split(":")
pars = {
'file_version': file_version,
'targetHost': get_fully_qualified_name(ngams_server),
'targetLocation': target_filename,
'file_id': file_id
}
fetch_start_time = time.time()
response = ngamsHttpUtils.httpGet(host,
int(port),
'RSYNC',
pars=pars,
timeout=rx_timeout)
with contextlib.closing(response):
data = response.read()
if 'FAILURE' in data:
raise Exception(data)
fetch_duration = time.time() - fetch_start_time
# Avoid divide by zeros later on, let's say it took us 1 [us] to do this
if fetch_duration == 0.0:
fetch_duration = 0.000001
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. Rate: %.2f Bytes/s"
logger.info(
msg, target_filename, int(request_properties.getBytesReceived()),
fetch_duration,
(float(request_properties.getBytesReceived()) / fetch_duration))
# Now check the CRC value against what we expected
checksum = request_properties.checksum
crc_variant = request_properties.checksum_plugin
crc_start_time = time.time()
crc = ngamsFileUtils.get_checksum(65536, target_filename, crc_variant)
crc_duration = time.time() - crc_start_time
logger.info("CRC computed in %f [s]", crc_duration)
logger.info('Cource checksum: %s - current checksum: %d', checksum, crc)
if crc != int(checksum):
msg = "Checksum mismatch: source={:s}, received={:d}".format(
checksum, crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
# We half the total time for reading and writing because we do not have enough data for an accurate measurement
read_duration = fetch_duration / 2.0
write_duration = fetch_duration / 2.0
read_total_bytes = request_properties.getBytesReceived()
return archiving_results(read_total_bytes, read_duration, write_duration,
crc_duration, fetch_duration, crc_variant, crc)
def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId):
"""
Execute CLONE Command, where the source Disk ID, File ID and File Version
are specified. Is much faster than a normal CLONE Command when an explicit
file is specified.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
diskId: ID of disk hosting the file to be cloned (string).
fileId: ID of file to clone (string).
fileVersion: Version of file to clone (integer).
targetDiskId: ID of target disk (string).
Returns: Void.
"""
T = TRACE(1)
# Resolve the location of the file to clone.
location, hostId, ipAddress, portNo, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
diskId=diskId,
fileVersion=fileVersion)
# Read also the entire file info (unfortunately).
srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(),
srvObj.getDb(), fileId,
fileVersion, diskId)
# Determine target disk.
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [diskId]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
# Don't accept to clone onto the same disk (this would meann overwriting).
if (trgDiskInfo.getDiskId() == diskId):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fileId +\
"/Version: " + str(fileVersion) +\
" on disk with ID: " + diskId +\
" on host: " + hostId + ". Reason: " + err
raise Exception, msg
# Receive the file into the staging filename.
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(mimeType)
stagingFilename = ngamsHighLevelLib.genStagingFilename(
srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId)
try:
quickLocation = False
if (reqPropsObj.hasHttpPar("quick")):
quickLocation = int(reqPropsObj.getHttpPar("quick"))
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?disk_id=%s&"
# file_id=id&file_version=1
fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\
"file_version=%s"
fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId,
str(fileVersion))
# If CLONE?quick specified, we try to retrieve the file via the
# RETRIEVE?quick_location method.
quickFileUrl = fileUrl
if (reqPropsObj.hasHttpPar("quick")):
if (int(reqPropsObj.getHttpPar("quick"))):
quickFileUrl = fileUrl + "&quick_location=1"
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
# TODO: a time-bomb waiting to explode....
fileUrl = "file:" + mtPt + "/" + filename
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
if (attempt == 0):
filename, headers = urllib.urlretrieve(
quickFileUrl, stagingFilename)
else:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, srcFileInfo, stagingFilename, headers, True)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(0.5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(srcFileInfo.getFilename())
targFilename = os.path.basename(srcFileInfo.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" + targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(srcFileInfo.getFileSize()).\
setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fileId, fileVersion, diskId, hostId])
logger.info(msg, extra={'to_syslog': True})
def getDiskInfo(srvObj, reqPropsObj=None):
"""
Invokes the plug-in listed in the configuration to extract the information
about the disks mounted. The information is returned in a dictionary which
contains the Slot IDs as keys, and elements which each is an object of type
ngamsPhysDiskInfo.
srvObj: Reference to NG/AMS Server class instance (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Dictionary containing information about the
disks (dictionary).
"""
T = TRACE()
diskInfoDic = {}
plugIn = srvObj.getCfg().getOnlinePlugIn()
if (not srvObj.getCfg().getSimulation()):
logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
diskInfoDic = plugInMethod(srvObj, reqPropsObj)
if (len(diskInfoDic) == 0):
if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())):
errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL",
[srvObj.getHostId(),
srvObj.getHostId()])
logger.warning(errMsg)
else:
if (srvObj.getCfg().getAllowArchiveReq()):
logger.debug("Running as a simulated archiving system - generating " +\
"simulated disk info ...")
portNo = 0
slotIdLst = srvObj.getCfg().getSlotIds()
for slotId in slotIdLst:
if (slotId != ""):
storageSet = srvObj.getCfg().\
getStorageSetFromSlotId(slotId)
if (storageSet.getMainDiskSlotId() == slotId):
diskType = "Main"
else:
diskType = "Rep"
mtRootDir = srvObj.getCfg().getRootDirectory()
mtPt = os.path.normpath(mtRootDir +\
"/"+storageSet.getStorageSetId() +\
"-" + diskType + "-" + str(slotId))
diskId = re.sub("/", "-", mtPt)[1:]
serialNo = "SERIAL-NUMBER-" + str(portNo)
manufact = "TEST-MANUFACTURER"
diskInfoDic[slotId] = ngamsPhysDiskInfo.ngamsPhysDiskInfo()
diskInfoDic[slotId].\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus("OK").\
setCapacityGb(100).\
setModel("TEST-MODEL").\
setSerialNo(serialNo).\
setType("TEST-TYPE").\
setManufacturer(manufact).\
setDiskId(diskId).\
setDeviceName("/dev/dummy" + slotId)
# We create the mount directory if it does not exist
checkCreatePath(mtPt)
portNo = portNo + 1
else:
# It is not an Archiving System. We check which of the directories
# in the Ngas.RootDirectory contain an NgasDiskInfo file.
# These are considered as available disks in simulation mode.
baseDir = srvObj.getCfg().getRootDirectory()
dirList = glob.glob(os.path.normpath(baseDir + "/*"))
for mtPt in dirList:
checkPath = os.path.normpath(mtPt + "/" + NGAMS_DISK_INFO)
logger.debug("Checking if path exists: %s", checkPath)
if (os.path.exists(checkPath)):
slotId = string.split(mtPt, "-")[-1]
portNo = (int(slotId) - 1)
diskId = re.sub("/", "-", mtPt)
serialNo = "SERIAL-NUMBER-" + str(portNo)
diskInfoDic[slotId]=ngamsPhysDiskInfo.ngamsPhysDiskInfo().\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus("OK").\
setCapacityGb(100).\
setModel("TEST-MODEL").\
setSerialNo(serialNo).\
setType("TEST-TYPE").\
setManufacturer("TEST-MANUFACTURER").\
setDiskId(diskId).\
setDeviceName("/dev/dummy" + slotId)
logger.debug("Disk Dictionary: %s", str(diskInfoDic))
return diskInfoDic
def save_to_file(ngams_server, request_properties, target_filename, block_size,
start_byte):
"""
Save the data available on an HTTP channel into the given file
:param ngams_server: Reference to NG/AMS server class object (ngamsServer)
:param request_properties: NG/AMS Request Properties object (ngamsReqProps)
:param target_filename: Target name for file where data will be written (string)
:param block_size: Block size (bytes) to apply when reading the data from the HTTP channel (integer)
:param start_byte: Start byte offset
:return: Tuple. Element 0: Time in took to write file (s) (tuple)
"""
disk_id = request_properties.fileinfo['diskId']
source_host = request_properties.fileinfo['sourceHost']
host_id = request_properties.fileinfo['hostId']
file_version = request_properties.fileinfo['fileVersion']
file_id = request_properties.fileinfo['fileId']
checksum = request_properties.checksum
crc_variant = request_properties.checksum_plugin
host, port = source_host.split(":")
parameter_list = [('disk_id', disk_id), ('host_id', host_id),
('quick_location', '1'), ('file_version', file_version),
('file_id', file_id)]
header_dict = {'Range': "bytes={:d}-".format(start_byte)}
rx_timeout = 30 * 60
if ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(
ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"))
url = 'http://{0}:{1}/{2}'.format(host, port, NGAMS_RETRIEVE_CMD)
authorization_header = ngamsSrvUtils.genIntAuthHdr(ngams_server)
response = ngamsHttpUtils.httpGetUrl(url, parameter_list, header_dict,
rx_timeout, authorization_header)
# Can we resume a previous download?
download_resume_supported = 'bytes' in response.getheader(
"Accept-Ranges", '')
logger.debug("Creating path: %s", target_filename)
checkCreatePath(os.path.dirname(target_filename))
logger.info('Fetching file ID %s, checksum %s, checksum variant %s',
file_id, checksum, crc_variant)
crc_info = ngamsFileUtils.get_checksum_info(crc_variant)
if start_byte != 0:
logger.info("Resume requested from start byte %d", start_byte)
if start_byte != 0 and download_resume_supported:
logger.info(
"Resume requested and mirroring source supports resume. Appending data to previous staging file"
)
crc = ngamsFileUtils.get_checksum(65536, target_filename, crc_variant)
request_properties.setBytesReceived(start_byte)
fd_out = open(target_filename, "ab")
else:
if start_byte > 0:
logger.info(
"Resume of download requested but server does not support it. Starting from byte 0 again."
)
fd_out = open(target_filename, "wb")
crc = crc_info.init
fetch_start_time = time.time()
# Distinguish between archive pull and push request
# By archive pull we may simply read the file descriptor until it returns and empty string
response_header_dict = {h[0].lower(): h[1] for h in response.getheaders()}
if 'content-length' in response_header_dict:
remaining_size = int(response_header_dict['content-length'])
logger.debug("Got Content-Length header value %d in response",
remaining_size)
else:
logger.warning(
"No Content-Length header found in response. Defaulting to 1e11")
remaining_size = int(1e11)
# Receive the data
read_size = block_size
crc_duration = 0
read_duration = 0
write_duration = 0
read_total_bytes = 0
crc_method = crc_info.method
with contextlib.closing(response), contextlib.closing(fd_out):
while remaining_size > 0:
if remaining_size < read_size:
read_size = remaining_size
# Read the remote file
read_start_time = time.time()
data_buffer = response.read(read_size)
read_duration += time.time() - read_start_time
size_read = len(data_buffer)
read_total_bytes += size_read
if size_read == 0:
raise ngamsFailedDownloadException.FailedDownloadException(
"server is unreachable")
# CRC
crc_start_time = time.time()
crc = crc_method(data_buffer, crc)
crc_duration += time.time() - crc_start_time
remaining_size -= size_read
request_properties.setBytesReceived(
request_properties.getBytesReceived() + size_read)
# Write the file onto disk
write_start_time = time.time()
fd_out.write(data_buffer)
write_duration += time.time() - write_start_time
crc = crc_info.final(crc)
fetch_duration = time.time() - fetch_start_time
# Avoid divide by zeros later on, let's say it took us 1 [us] to do this
if fetch_duration == 0.0:
fetch_duration = 0.000001
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. Rate: %.2f Bytes/s"
logger.info(
msg, target_filename, int(request_properties.getBytesReceived()),
fetch_duration,
(float(request_properties.getBytesReceived()) / fetch_duration))
# Raise exception if bytes received were less than expected
if remaining_size != 0:
msg = "Not all expected data arrived, {:d} bytes left to read".format(
remaining_size)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
# Now check the freshly calculated CRC value against the stored CRC value
logger.info('Source checksum: %s - received checksum: %d', checksum, crc)
if not crc_info.equals(checksum, crc):
msg = "checksum mismatch: source={:s}, received={:d}".format(
checksum, crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
return archiving_results(read_total_bytes, read_duration, write_duration,
crc_duration, fetch_duration, crc_variant, crc)
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
T = TRACE(1)
cloneListDbm = cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr=0,
writePerm=1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.getThreadRunPermission()): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception, errMsg
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug(
"Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" +
targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVer, filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId
])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception, e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId,
str(e)
])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
thread.exit()
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
def _dumpFileInfo(srvObj, disks_to_check, tmpFilePat, stopEvt):
"""
Function that dumps the information about the files. One DBM is created
per disk. This is named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_QUEUE_<Disk ID>.bsddb
If problems are found for a file, these are stored in DBM files named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_ERRORS_<Disk ID>.bsddb
The function handles the DBM files in the following way:
1. Check for each DBM file found, if this disk is still in the system.
If not, the Queue and Error DBM files are removed.
2. Go through the list of disks in the system. If they don't have
the two DBM files listed above, these are initialized. The file
information for all the files stored on the disk is dumped into
the Queue DBM file. Only files marked to be ignored are not dumped.
3. Finally, build up a DBM with references to all files found
on this system
srvObj: Reference to server object (ngamsServer).
tmpFilePat: Pattern for temporary files (string).
Returns: Void.
"""
T = TRACE()
cacheDir = os.path.join(srvObj.getCfg().getRootDirectory(),
NGAMS_CACHE_DIR)
checkCreatePath(os.path.normpath(cacheDir))
###########################################################################
# Loop over the Queue/Error DBM files found, check if the disk is
# still in the system/scheduled for checking.
###########################################################################
logger.debug("Loop over/check existing Queue/Error DBM Files ...")
dbmFileList = glob.glob(cacheDir + "/" + NGAMS_DATA_CHECK_THR +\
"_QUEUE_*.bsddb")
dbmObjDic = {}
for dbmFile in dbmFileList:
_stopDataCheckThr(stopEvt)
diskId = dbmFile.split("_")[-1].split(".")[0]
if diskId not in disks_to_check:
filePat = "%s/%s*%s.bsddb" % (cacheDir, NGAMS_DATA_CHECK_THR,
diskId)
rmFile(filePat)
else:
# Add references to Queue/Error DBM.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
logger.debug("Looped over/checked existing Queue/Error DBM Files")
###########################################################################
###########################################################################
# Loop over the disks mounted in this system and check if they have a
# Queue/Error DBM file. In case the DBM files are not available, create
# these.
###########################################################################
logger.debug("Create DBM files for disks to be checked ...")
startDbFileRd = time.time()
for diskId in disks_to_check.keys():
_stopDataCheckThr(stopEvt)
if diskId in dbmObjDic:
continue
# The disk is ripe for checking but still has no Queue/Error DBM
# DBs allocated.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
tmpQueueDbmFile = tmpFilePat + "_" + os.path.basename(queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(tmpQueueDbmFile, 0, 1)
# Now, retrieve the files on the given disk, and store the info
# in the Queue DBM file.
files = srvObj.getDb().getFileSummary1(diskIds=[diskId],
ignore=0,
fileStatus=[],
lowLimIngestDate=None,
order=0)
for fileInfo in files:
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVer)
queueDbm.add(fileKey, fileInfo)
queueDbm.sync()
# Rename DCC Queue DBM from the temporary to the final name.
mvFile(tmpQueueDbmFile, queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
# Create Error DBM + add these in the DBM Dictionary for the disk.
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
_stopDataCheckThr(stopEvt)
logger.debug("Queried info for files to be checked from DB. Time: %.3fs",
time.time() - startDbFileRd)
logger.debug("Checked that disks scheduled for checking have DBM files")
###########################################################################
# These are all files recursively found on the disks
# Later on we check whether they are registered or not, and check them (or not)
start = time.time()
files_on_disk = collect_files_on_disk(stopEvt, disks_to_check)
end = time.time()
logger.debug("Collected references to %d files on disks in %.3f [s]",
len(files_on_disk), end - start)
# Don't take these into account
logger.debug("Retrieving information about files to be ignored ...")
files = srvObj.getDb().getFileSummarySpuriousFiles1(srvObj.getHostId(),
fetch_size=1000)
for fileInfo in files:
if (fileInfo[ngamsDbCore.SUM1_FILE_IGNORE]):
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] + "/" +\
fileInfo[ngamsDbCore.SUM1_FILENAME])
if filename in files_on_disk:
del files_on_disk[filename]
logger.debug("Retrieved information about files to be ignored")
###########################################################################
###########################################################################
# Initialize the statistics parameters for the checking.
###########################################################################
logger.debug("Initialize the statistics for the checking cycle ...")
amountMb = 0.0
noOfFiles = 0
for diskId in disks_to_check.keys():
queueDbm = dbmObjDic[diskId][0]
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#queueDbm.initKeyPtr()
#while (1):
# fileKey, fileInfo = queueDbm.getNext()
# if (not fileKey): break
for fileKey, dbVal in queueDbm.iteritems():
# jagonzal: We need to reformat the values and skip administrative elements #################
if (str(fileKey).find("__") != -1): continue
fileInfo = cPickle.loads(dbVal)
#############################################################################################
noOfFiles += 1
amountMb += float(fileInfo[ngamsDbCore.SUM1_FILE_SIZE]) / 1048576.0
#################################################################################################
stats = _initFileCheckStatus(srvObj, amountMb, noOfFiles)
###########################################################################
return files_on_disk, dbmObjDic, stats
def bbcpFile(srcFilename, targFilename, bparam, crc_name, skip_crc):
"""
Use bbcp tp copy file <srcFilename> to <targFilename>
NOTE: This requires remote access to the host as well as
a bbcp installation on both the remote and local host.
"""
logger.debug("Copying file: %s to filename: %s", srcFilename, targFilename)
# Make target file writable if existing.
if (os.path.exists(targFilename)):
os.chmod(targFilename, 420)
checkCreatePath(os.path.dirname(targFilename))
if bparam.port:
pt = ['-Z', str(bparam.port)]
else:
pt = ['-z']
fw = []
if bparam.winsize:
fw = ['-w', str(bparam.winsize)]
ns = []
if (bparam.num_streams):
ns = ['-s', str(bparam.num_streams)]
# bypass password prompt with -oBatchMode=yes this implies you need keys
ssh_src = [
'-S',
'ssh -x -a -oBatchMode=yes -oFallBackToRsh=no %4 %I -l %U %H bbcp'
]
# perform checksum on host and compare to target. If it's different bbcp will fail.
if not skip_crc and crc_name is not None:
cmd_checksum = ['-e', '-E']
if crc_name in ('crc32', 'crc32z'):
# c32z is the zip-flavor of CRC32
# c32 is the POSIX flavour, which yields a different result
cmd_checksum.append('c32z=/dev/stdout')
elif crc_name == 'crc32c':
cmd_checksum.append('c32c=/dev/stdout')
else:
raise Exception("Unsupported checksum method in BBCP: %s" %
(crc_name, ))
cmd_list = ['bbcp', '-f', '-V'] + ssh_src + cmd_checksum + fw + ns + [
'-P', '2'
] + pt + [srcFilename, targFilename]
logger.info("Executing external command: %s",
subprocess.list2cmdline(cmd_list))
p1 = subprocess.Popen(cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
checksum_out, out = p1.communicate()
if p1.returncode != 0:
raise Exception("bbcp returncode: %d error: %s" % (p1.returncode, out))
# extract c32 zip variant checksum from output and convert to signed 32 bit integer
crc_info = ngamsFileUtils.get_checksum_info(crc_name)
checksum_bytes = codecs.decode(checksum_out.split(b' ')[2], 'hex')
bbcp_checksum = crc_info.from_bytes(checksum_bytes)
logger.info(
'BBCP final message: %s',
out.split(b'\n')[-2]) # e.g. "1 file copied at effectively 18.9 MB/s"
logger.info("File: %s copied to filename: %s", srcFilename, targFilename)
return str(bbcp_checksum)
def _cloneExec(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr = 0, writePerm = 1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.run_async_commands): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception(errMsg)
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception as e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception(errMsg)
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/"+targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
fileId, fileVer,
filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception as e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId, str(e)])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
return
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
# Calculate time statistics.
if (reqPropsObj):
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
incActualCount(1), timeAccu)
# Final update of the Request Status.
if (reqPropsObj):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Clone Report with list of files cloned to a possible
# requestor(select) of this.
totFiles = (successCloneCount + failedCloneCount)
if (emailNotif):
xmlStat = 0
# TODO: Generation of XML status report is disabled since we cannot
# handle for the moment XML documents with 1000s of elements.
if (xmlStat):
cloneStatusFileList = ngamsFileList.\
ngamsFileList("FILE_CLONING_STATUS_REPORT",
"File Cloning Status Report")
fileCount = 0
while (fileCount < cloneStatusDbm.getCount()):
tmpFileList = cloneStatusDbm.get(str(fileCount))
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
else:
fileVersion = "-----"
tmpFormat = "CLONE STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Disk ID: %s\n" +\
"File ID: %s\n" +\
"File Version: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Cloned Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
str(fileVersion), totFiles,
successCloneCount, failedCloneCount,
timeAccu, (timeAccu / totFiles)))
tmpFormat = "%-70s %-70s %-7s\n"
fo.write(tmpFormat % ("Source File", "Target File", "Status"))
fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
key = 1
while (1):
if (not cloneStatusDbm.hasKey(str(key))): break
tmpFileList = cloneStatusDbm.get(str(key))
key += 1
srcFileObj = tmpFileList.getFileInfoObjList()[0]
srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
srcFileObj.getFileId(),
srcFileObj.getFileVersion())
if (tmpFileList.getStatus() == NGAMS_SUCCESS):
trgFileObj = tmpFileList.getFileInfoObjList()[1]
trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
trgFileObj.getFileId(),
trgFileObj.getFileVersion())
else:
trgFile = "-----"
fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
fo.write(149 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "CloneStatusReport"
if (reqPropsObj.hasHttpPar("disk_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
"CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
contentType=mimeType, attachmentName=attachmentName)
del cloneStatusDbm
rmFile(cloneStatusDbmName + "*")
rmFile(statRep)
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
timeAccu, (timeAccu / totFiles))
def _createDirectories(self):
for mydir in self.mydirs:
checkCreatePath(mydir)
def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = "/tmp/ngamsTest/NGAS/"
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(['tar', 'zxf', 'src/volumes_dir.tar.gz'])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
cwd = os.getcwd()
configFile = os.path.normpath(cwd +
"/src/ngamsCfg_VolumeDirectory.xml")
self.prepExtSrv(delDirs=0, cfgFile=configFile)
client = sendPclCmd()
# Archive a file.
stat = client.archive("src/SmallFile.fits")
# dpallot: this will always fail on the mac as the tar sizes are different
# to the hard coded test results in the old file:
# ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref
#tmpStatFile = saveInFile(None, filterDbStatus1(stat.dumpBuf()))
#refStatFile = "ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref"
#self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
# "message from NG/AMS Server")
self.assertEquals(stat.getStatus(), 'SUCCESS')
# Check that the target files have been archived in their
# appropriate locations.
trgFile = "tmp/test_VolumeDir_01_tmp"
refFile = "src/SmallFile.fits"
outFilePath = "tmp/SmallFile.fits"
stat = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
self.assertEqual(NGAMS_SUCCESS, stat.getStatus(), stat.getMessage())
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(outFilePath, refFile, "Retrieved file incorrect")
def saveToFile(srvObj, ngamsCfgObj, reqPropsObj, trgFilename, blockSize,
startByte):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
source_host = reqPropsObj.fileinfo['sourceHost']
file_version = reqPropsObj.fileinfo['fileVersion']
file_id = reqPropsObj.fileinfo['fileId']
logger.info("Creating path: %s", trgFilename)
checkCreatePath(os.path.dirname(trgFilename))
rx_timeout = 30 * 60
if srvObj.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(srvObj.getCfg().getVal("Mirroring[1].rx_timeout"))
host, port = source_host.split(":")
pars = {
'file_version': file_version,
'targetHost': get_full_qualified_name(srvObj),
'targetLocation': trgFilename,
'file_id': file_id
}
start = time.time()
response = ngamsHttpUtils.httpGet(host,
int(port),
'RSYNC',
pars=pars,
timeout=rx_timeout)
with contextlib.closing(response):
data = response.read()
if 'FAILURE' in data:
raise Exception(data)
deltaTime = time.time() - start
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. " +\
"Rate: %.2f Bytes/s"
logger.info(msg, trgFilename, int(reqPropsObj.getBytesReceived()),
deltaTime, (float(reqPropsObj.getBytesReceived()) / deltaTime))
# now check the CRC value against what we expected
sourceChecksum = reqPropsObj.checksum
start = time.time()
crc = ngamsFileUtils.get_checksum(65536, trgFilename, 'crc32')
deltaTime = time.time() - start
logger.info("crc computed in %f [s]", deltaTime)
logger.info('source checksum: %s - current checksum: %d',
str(sourceChecksum), crc)
if (crc != int(sourceChecksum)):
msg = "checksum mismatch: source=" + str(
sourceChecksum) + ", received: " + str(crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
return [deltaTime, crc]
# EOF
def create_test_sample_file(self, file_name, sample):
file_path = self.ngas_path(TEST_PATH + file_name)
checkCreatePath(os.path.dirname(file_path))
with open(file_path, 'w') as fo:
fo.write(sample)
return self.get_status(NGAMS_REGISTER_CMD, (("path", file_path), ))
def _handleCmdRetrieve(srvObj, reqPropsObj, httpRef):
"""
Carry out the action of a RETRIEVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of
actions done during the request handling
(ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# For data files, retrieval must be enabled otherwise the request is
# rejected.
if (not srvObj.getCfg().getAllowRetrieveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"])
raise Exception(errMsg)
# Previously this command allowed to retrieve the current logging file,
# the configuration file and any internal file. We don't do this anymore
# Get query information.
if 'ng_log' in reqPropsObj or 'cfg' in reqPropsObj or 'internal' in reqPropsObj:
raise Exception(
"ng_log, cfg and internal parameters not supported anymore")
# At least file_id must be specified if not an internal file has been
# requested.
if 'file_id' not in reqPropsObj or not reqPropsObj['file_id']:
errMsg = genLog("NGAMS_ER_RETRIEVE_CMD")
raise Exception(errMsg)
fileId = reqPropsObj.getHttpPar("file_id")
logger.debug("Handling request for file with ID: %s", fileId)
fileVer = -1
if (reqPropsObj.hasHttpPar("file_version")):
fileVer = int(reqPropsObj.getHttpPar("file_version"))
diskId = ""
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
hostId = ""
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
domain = ""
if (reqPropsObj.hasHttpPar("domain")):
domain = reqPropsObj.getHttpPar("domain")
quickLocation = True
if (reqPropsObj.hasHttpPar("quick_location")):
quickLocation = int(reqPropsObj.getHttpPar("quick_location"))
# First try the quick retrieve attempt, just try to get the first
# (and best?) suitable file which is online and located on a node in the
# same domain as the contacted node.
ipAddress = None
if (quickLocation):
location, host, ipAddress, port, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
hostId, domain, diskId,
fileVer)
# If not located the quick way try the normal way.
if (not ipAddress):
# Locate the file best suiting the query and send it back if possible.
location, host, ipAddress, port, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVer,
diskId, hostId, reqPropsObj)
# If still not located, try to contact associated NGAS sites to query
# if the file is available there.
# TODO:
if (not ipAddress):
pass
if (location == NGAMS_HOST_LOCAL):
# Get the file and send back the contents from this NGAS host.
srcFilename = os.path.normpath("{0}/{1}".format(mountPoint, filename))
# Perform the possible file staging
performStaging(srvObj, reqPropsObj, httpRef, srcFilename)
# Perform the possible processing requested.
procResult = performProcessing(srvObj, reqPropsObj, srcFilename,
mimeType)
elif location in (NGAMS_HOST_CLUSTER, NGAMS_HOST_REMOTE) and \
srvObj.getCfg().getProxyMode():
logger.debug("NG/AMS Server acting as proxy - requesting file with ID: %s " +\
"from NG/AMS Server on host/port: %s/%s",
fileId, host, str(port))
# Act as proxy - get the file from the NGAS host specified and
# send back the contents. The file is temporarily stored in the
# Processing Area.
procDir = ngamsHighLevelLib.genProcDirName(srvObj.getCfg())
checkCreatePath(procDir)
pars = []
for par in reqPropsObj.getHttpParNames():
pars.append([par, reqPropsObj.getHttpPar(par)])
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
timeout = float(
reqPropsObj['timeout']) if 'timeout' in reqPropsObj else 60
conn = ngamsHttpUtils.httpGet(ipAddress,
port,
NGAMS_RETRIEVE_CMD,
pars=pars,
timeout=timeout,
auth=authHdr)
hdrs = {h[0]: h[1] for h in conn.getheaders()}
dataSize = int(hdrs["content-length"])
tmpPars = ngamsLib.parseHttpHdr(hdrs["content-disposition"])
dataFilename = tmpPars["filename"]
data = ngamsHttpUtils.sizeaware(conn, dataSize)
httpRef.send_data(data, mimeType, fname=dataFilename)
return
else:
# No proxy mode: A redirection HTTP response is generated.
httpRef.redirect(ipAddress, port)
return
# Send back reply with the result(s) queried and possibly processed.
genReplyRetrieve(srvObj, reqPropsObj, httpRef, procResult)
def saveFromHttpToFile(ngamsCfgObj,
reqPropsObj,
httpRef,
trgFilename,
blockSize,
mutexDiskAccess=1,
diskInfoObj=None):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
T = TRACE()
checkCreatePath(os.path.dirname(trgFilename))
start = time.time()
try:
# Make mutual exclusion on disk access (if requested).
if (mutexDiskAccess):
ngamsHighLevelLib.acquireDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
remSize = reqPropsObj.getSize()
logger.debug("Archive Push/Pull Request - Data size: %d", remSize)
handler = ngamsMIMEMultipart.FilesystemWriterHandler(
blockSize, True, trgFilename)
parser = ngamsMIMEMultipart.MIMEMultipartParser(
handler, httpRef.rfile, remSize, blockSize)
parser.parse()
deltaTime = time.time() - start
fileDataList = handler.getFileDataList()
crcTime = handler.getCrcTime()
writingTime = handler.getWritingTime()
rootContainer = handler.getRoot()
readingTime = parser.getReadingTime()
bytesRead = parser.getBytesRead()
ingestRate = (float(bytesRead) / deltaTime)
reqPropsObj.setBytesReceived(bytesRead)
logger.debug(
"Transfer time: %.3f s; CRC time: %.3f s; write time %.3f s",
readingTime, crcTime, writingTime)
return [deltaTime, rootContainer, fileDataList, ingestRate]
finally:
# Release disk resouce.
if (mutexDiskAccess):
ngamsHighLevelLib.releaseDiskResource(ngamsCfgObj,
diskInfoObj.getSlotId())
