def _checkContDbSnapshot(testSuiteObj,
testCaseNo,
dataDirList,
waitEmptyCache=True,
filterContents=True):
"""
Function to check the contents of a DB Snapshot file.
testSuiteObj: Reference to Test Suite Object (ngamsTestSuite).
testCaseNo: Test Case number (integer).
dataDirList: List of directories under the NGAS Root Mount Point
to check (list).
Returns: Void.
"""
dirPat = testSuiteObj.ngas_path("%s/.db/NgasFiles.bsddb")
cacheDirPat = testSuiteObj.ngas_path("%s/.db/cache")
refFilePat = "ref/ngamsDbSnapShotTest_test_DbSnapshot_%d_%d.ref"
count = 1
startTime = time.time()
for dataDir in dataDirList:
# Wait till the DB Snapshot Cache Dir is empty.
if (waitEmptyCache):
cacheDir = cacheDirPat % dataDir
startTime = time.time()
while (((time.time() - startTime) < 10)
and (not os.path.exists(cacheDir))):
time.sleep(0.200)
startTime = time.time()
while ((time.time() - startTime) < 20):
tmpDbSnapshot = glob.glob(cacheDir + "/*")
if (len(tmpDbSnapshot) == 0): break
time.sleep(0.200)
complName = dirPat % dataDir
refFile = refFilePat % (testCaseNo, count)
startTime = time.time()
while ((not os.path.exists(complName))
and ((time.time() - startTime) < 10)):
time.sleep(0.200)
testSuiteObj.assertTrue(os.path.exists(complName),
"DB Snapshot missing: " + complName)
out = utils.b2s(
subprocess.check_output([
sys.executable, '-m', 'ngamsServer.ngamsDumpDbSnapshot',
complName
]))
if (filterContents):
snapshotDump = _parseDbSnapshot(out)
else:
snapshotDump = out
testSuiteObj.assert_ref_file(refFile,
snapshotDump,
msg="Incorrect contents of DB Snapshot")
count += 1
def _authorization_cfg(self, user_specs):
uspecs = [(('Name', u[0]), ('Password',
utils.b2s(base64.b64encode(u[1]))),
('Commands', u[2])) for u in user_specs]
cfg = [('Enable', '1')]
for i, u in enumerate(uspecs):
cfg += [('User[%d].%s' % (i, name), str(val)) for name, val in u]
cfg = [("NgamsCfg.Authorization[1].%s" % name, val)
for name, val in cfg]
return cfg
def test_UnAuthReq_2(self):
"""Authorization/authentication is enabled, commands are restricted"""
# Start the server with a set of configured users
pass1 = os.urandom(16)
pass2 = os.urandom(16)
pass3 = os.urandom(16)
u1 = (('Name', 'test1'),
('Password', utils.b2s(base64.b64encode(pass1))), ('Commands',
'*'))
u2 = (('Name', 'test2'),
('Password', utils.b2s(base64.b64encode(pass2))), ('Commands',
'STATUS'))
u3 = (('Name', 'test3'), ('Password',
utils.b2s(base64.b64encode(pass3))),
('Commands', 'DOESNT_EXIST'))
cfg = [('Enable', '1')]
for i, u in enumerate((u1, u2, u3)):
cfg += [('User[%d].%s' % (i, name), str(val)) for name, val in u]
cfg = [("NgamsCfg.Authorization[1].%s" % name, val)
for name, val in cfg]
self.prepExtSrv(cfgProps=cfg)
auth1 = b'test1:' + pass1
auth2 = b'test2:' + pass2
auth3 = b'test3:' + pass3
# No authentication
self._assert_code(401)
# Users 1 and 2 are allowed to STATUS, user 3 isn't
self._assert_code(200, bauth=auth1)
self._assert_code(200, bauth=auth2)
self._assert_code(403, bauth=auth3)
# Users 1 and 3 are allowed to send the DOESNT_EXIST command, user 3 isn't
# We get 404s here because the command actually doesn't exist, which yields
# that HTTP code, and also because HTTP authentication happens before
# command processing
self._assert_code(404, bauth=auth1, cmd='DOESNT_EXIST')
self._assert_code(403, bauth=auth2, cmd='DOESNT_EXIST')
self._assert_code(404, bauth=auth3, cmd='DOESNT_EXIST')
def loadVolInfoFile(fname):
"""Load an NGAS Volume Info file and return it as a dictionary"""
with open(fname, "rb") as fo:
volInfoBuf = utils.b2s(base64.b64decode(fo.read()))
dic = {}
for line in volInfoBuf.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
par, val = line.split("=")
dic[par.strip()] = val.strip()
return dic
def test_column_names(self):
"""Check that column names are correctly bound to data by reading some
of the cells and making sure they make sense. If column names are not
correclty bound to columns then data might not be convertible and might
not make sense"""
cfg, _ = self.prepExtSrv()
stat = self.assert_query(
pars=[['query', 'disks_list'], ['format', 'json']])
results = json.loads(utils.b2s(stat.getData()))
self.assertEqual(0, int(results[0]['number_of_files']))
self.assertEqual(cfg.getArchiveName(), results[0]['archive'])
def authorize(cfg, reqPropsObj):
"""Check if the request is authorized for the authenticated user, if any"""
if not cfg.getAuthorize():
logger.debug("Authorization is disabled, continuing anonymously")
return
# Check if the requested command is excluded from authorization
requested_command = reqPropsObj.getCmd().strip()
if isCommandExcluded(cfg, requested_command):
logger.debug(
"Authorization is disabled for the '%s' command, continuing anonymously",
requested_command)
return
# For now only Basic HTTP Authentication is implemented.
if not reqPropsObj.getAuthorization():
raise UnauthenticatedError('Unauthorized request received')
auth_parts = reqPropsObj.getAuthorization().split(' ')
if auth_parts[0] != 'Basic':
raise UnauthenticatedError('Invalid authentication scheme: ' +
auth_parts[0])
if len(auth_parts) != 2:
raise UnauthenticatedError(
'Invalid Basic authentication, missing value')
user_pass = base64.b64decode(auth_parts[1]).split(b':')
if len(user_pass) < 2:
raise UnauthenticatedError(
'Invalid Basic authentication, no password provided')
user, password = utils.b2s(user_pass[0]), b':'.join(user_pass[1:])
# Get the user from the configuration.
stored_pass = cfg.getAuthUserInfo(user)
if not stored_pass:
raise UnauthenticatedError("unknown user specified")
# Password matches and command is allowed
stored_pass = base64.b64decode(six.b(stored_pass))
if password != stored_pass:
raise UnauthenticatedError("wrong password for user " + user)
if not cmdPermitted(cfg, reqPropsObj, user):
raise UnauthorizedError(user)
logger.info("Successfully authenticated user %s", user)
def _execCClient(unpackXmlStat = 1,
pars = []):
"""
Execute the NG/AMS C-Client on the shell.
cmdLineParsDic: Dictionary with command line parameters (dictionary).
pars: Extra parameters for invoking NGAMS C-Client (list).
Returns: List with status objects or stdout output from c-client:
[<stat obj>, ...] or <stdout c-client> (list|string).
"""
cmd = ['ngamsCClient']
for opt, val in pars:
cmd.append(opt)
cmd.append(str(val))
if '-servers' not in (x for x, _ in pars):
cmd.append('-host')
cmd.append('127.0.0.1')
cmd.append('-status')
env = os.environ.copy()
env['NGAMS_VERBOSE_LEVEL'] = '0'
_, out, _ = execCmd(cmd, shell=False)
out = utils.b2s(out)
if (unpackXmlStat):
statObjList = []
xmlStatList = out.split("Command repetition counter:")
for xmlStat in xmlStatList:
# Clean up the output.
xmlStat = xmlStat.strip()
if (xmlStat == ""): continue
idx = 0
while (xmlStat[idx] != "<"): idx += 1
xmlStat = xmlStat[idx:]
# Unpack it.
statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat)
statObjList.append(statObj)
out = statObjList
return out
def readDMFStatus(filename):
"""
Lists file names in long format, giving
mode, number of links, owner, group, size in bytes, time of last modification, and,
in parentheses before the file name, the DMF state
"""
cmd = ['dmls', '-l', filename]
proc = Popen(cmd, stdout=PIPE, close_fds=True)
out, err = proc.communicate()
if proc.returncode != 0:
raise Exception(out)
try:
out = utils.b2s(out)
status = out.split()[7][1:4]
if status in ERROR_STATUS:
raise Exception('dmls status error: %s filename: %s' %
(status, filename))
return status
except IndexError as e:
raise Exception('dmls output error')
def loadVolInfoFile(volInfoFile):
"""
Load the contents of a NGAS Volume Info file and return a dictionary
with the parameters.
volInfoFile: Path of Volume Info File (string).
Returns: Dictionary with the parameters of the Volume Info File
(dictionary).
"""
with open(volInfoFile, "rb") as fo:
volInfoBuf = utils.b2s(base64.decodestring(fo.read()))
dic = {}
for line in volInfoBuf.split("\n"):
line = line.strip()
if ((line == "") or (line[0] == "#")): continue
par, val = line.split("=")
dic[par.strip()] = val.strip()
return dic
def dumpDbSnapshot(dbSnapshotName, details=0):
"""
Dump the contents of a DB Snapshot in a raw format.
dbSnapshotName: Name of DB Snapshot (string).
details: Dump details (integer/0|1).
Returns: Void.
"""
db = bsddb.hashopen(dbSnapshotName, "r")
print("\nDumping contents of NG/AMS DB Snapshot: " + dbSnapshotName)
try:
key, pickleVal = db.first()
except:
key = None
print("")
while (key):
try:
val = cPickle.loads(pickleVal)
except Exception as e:
tstStr = "No module named"
if (str(e).find(tstStr) != -1):
modName = str(e).split(tstStr)[-1].strip()
exec("import %s" % modName)
val = cPickle.loads(pickleVal)
else:
raise
# Try if the object has a dumpBuf() method.
try:
dumpBuf = val.dumpBuf()
except:
dumpBuf = None
print("%s = %s" % (utils.b2s(key), str(val)))
if (details and dumpBuf): print(dumpBuf)
try:
key, pickleVal = db.next()
except:
break
db.close()
def send_mirrexec_command(self):
"""
Send MIRREXEC command to the input source_node
"""
logger.info("Sending MIRREXEC command to %s with (n_threads=%d)",
self.target_node, self.num_threads)
try:
host, port = self.target_node.split(":")
pars = {
'n_threads': str(self.num_threads),
'rx_timeout': str(self.rx_timeout),
'iteration': str(self.iteration)
}
start_time = time.time()
# it is important to not let this operation time out. If it times out then the files being fetched will
# be eligable for re-fetching even though the spawned threads may still be executing. Chaos ensues.
response = ngamsHttpUtils.httpGet(host,
int(port),
'MIRREXEC',
pars=pars)
with contextlib.closing(response):
failed = response.status != NGAMS_HTTP_SUCCESS or NGAMS_FAILURE in utils.b2s(
response.read())
elapsed_time = time.time() - start_time
if failed:
logger.error(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status FAILURE "
"in %f [s]", self.target_node, self.num_threads,
elapsed_time)
else:
logger.info(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status SUCCESS in %f [s]",
self.target_node, self.num_threads, elapsed_time)
except Exception:
logger.exception("Problems sending MIRREXEC command to %s",
self.target_node)
return
def checkFileCopiesAndReg(srvObj,
minReqCopies,
dbFilePat,
fileListDbmName=None,
diskId=None,
ignoreMounted=0):
"""
The function checks for each file referenced if there are at least
'minReqCopies' copies available somewhere in this NGAS cluster. For the
files where this is not the case, an entry is added in a ngasDiskInfo
object indicating that this file
If an entire disk is analysed (about to be deleted), it is also checked if
each file stored on the disk is registered in the DB. Otherwise an NG/AMS
File List is returned, containing references to the files not registered.
srvObj: Instance of the NG/AMS Server Class (ngamsServer).
minReqCopies: Minimum number of copies required (integer).
dbFilePat: Filename pattern used to build the DBM containing
information about the files (string).
fileListDbmName: Name of DBM DB containing explicit references to files
to be checked if they can be deleted. The information
in this table is pickled lists with the lay-out defined
by ngamsDb._ngasFilesCols (string).
diskId: Used to refer to all files stored on a disk (string|None).
ignoreMounted: Carry out the check also if the disk is not mounted
(integer/0|1).
Returns: Tuple contaning the filenames of three DBM DBs with the
following information:
o Files not having the specified number of copies.
The contents of this DB are keys (Disk ID + File ID +
File Version), pointing to pickled ngamsFileInfo
objects.
o Files found on the disk but not registered.
This contains the complete filenames of files found
on the disk, which are not registered. These filenames
are the keys of this DBM DB.
o Complete list of files referenced in connection
with the query. The contents of this DB are keys,
which are a simple counter pointing to pickled
list containing the information as returned by
ngamsDb.getFileSummary1().
(tuple/string).
"""
T = TRACE()
if ((not fileListDbmName) and (not diskId)):
errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\
"either a DBM with files to be checked or a Disk ID"
logger.warning(errMsg)
raise Exception(errMsg)
# Create DBMs:
# - DB containing information about files having less then the
# specified number of copies.
fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES")
fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1)
# - DB that contains information about files stored on the DB,
# which are not registered in the NGAS DB. At the end of the function,
# this will contain information about files found on the disk but
# not registered in the NGAS DB.
filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK")
filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1)
# - DB with information about files referenced by the query.
if (not fileListDbmName):
locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST")
fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1)
else:
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0)
# - Temporary DBM containing information about all File IDs defined
# by the query.
fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS")
fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1)
# - Temporary DBM containing information about all files available in
# the system with the File ID/File Version defined by the query.
complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST")
complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1)
# - Temporary DBM that is used to figure out the number of independent
# copies of each file concerned by the query.
checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC")
checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1)
# A Disk ID but no file references are given. Retrieve information
# about files concerned from the DB.
if (diskId):
logger.debug("Retrieving information about files on disk with ID: %s",
diskId)
for f in srvObj.db.getFileSummary1(None, [diskId], [],
ignore=0,
fileStatus=[]):
fileListDbm.addIncKey(f)
fileId = f[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
fileVersion = f[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileListDbm.sync()
fileIdDbm.sync()
checkDicDbm.sync()
# Get the list of files located on the disk. Later on, remove entries
# from this dictionary as the files are parsed, based on their DB info,
# further down in this method.
#
# Key in this dictionary is the complete filename of the file.
logger.debug("Get list of files stored on disk ...")
tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId)
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(tmpDiskInfo)
if ((not ignoreMounted) and (not diskInfoObj.getMounted())):
errMsg = "Rejecting request for removing disk with ID: " +\
diskId + " - disk not mounted!"
raise Exception(errMsg)
if (not ignoreMounted):
basePath = os.path.normpath(diskInfoObj.getMountPoint())
pattern = "/*"
logger.debug(
"Generating list with files on disk with base path: %s",
basePath)
while (1):
tmpFileList = glob.glob(basePath + pattern)
if (len(tmpFileList) == 0):
break
else:
for filename in tmpFileList:
if (os.path.isfile(filename) and
(os.path.basename(filename) != NGAMS_DISK_INFO)
and (os.path.basename(filename) !=
NGAMS_VOLUME_ID_FILE)
and (os.path.basename(filename) !=
NGAMS_VOLUME_INFO_FILE)):
filesOnDiskDicDbm.add(str(filename), "")
pattern += "/*"
# Generate File ID DBM in case a file list DBM is given.
if (fileListDbmName):
logger.debug("Handling file list DBM given in the function call ...")
fileListDbm.initKeyPtr()
while (1):
key, tmpFileInfo = fileListDbm.getNext()
if (not key): break
# Update the File ID DBM.
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileIdDbm.add(str(fileId), "")
# Update the DBM with references to File ID/Version sets.
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
checkDicDbm.add(fileKey, {})
fileIdDbm.sync()
checkDicDbm.sync()
# We need to generate a list with all files available in the system
# with the given File ID/File Version.
logger.debug("Retrieving information about all files available with the " +\
"File ID/File Version as defined by the query")
# Due to the limitation of the size of SQL queries, we have to split up
# the SQL query in several sub-queries. The max. length of an SQL query
# is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for
# the general part of the query, and for each filename we calculate a
# length of len(File ID) + 4 as contribution to the SQL query.
maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512)
queryIds = []
querySize = 0
noOfFileIds = fileIdDbm.getCount()
fileIdCount = 0
fileIdDbm.initKeyPtr()
fileId = "INIT"
while (fileId):
fileId, dummy = fileIdDbm.getNext()
if (fileId):
queryIds.append(utils.b2s(fileId))
fileIdCount += 1
querySize += (len(fileId) + 4)
if queryIds and (querySize >= maxQuerySize
or fileIdCount == noOfFileIds):
for f in srvObj.db.getFileSummary1(fileIds=queryIds,
fileStatus=[]):
# Take only a sub-result if that File ID + Version
# is concerned by the query.
tmpFileId = f[ngamsDbCore.SUM1_FILE_ID]
tmpFileVersion = f[ngamsDbCore.SUM1_VERSION]
tmpFileKey = ngamsLib.genFileKey(None, tmpFileId,
tmpFileVersion)
if (checkDicDbm.hasKey(tmpFileKey)):
complFileListDbm.addIncKey(f)
complFileListDbm.sync()
queryIds = []
querySize = 0
# Now, go through the files found and order these such that we end up with
# a Dictionary with "<File ID>_<File Version>" as keys referring
# to a dictionary with the Disk IDs of the disks hosting the files as
# keys, and the information for each file on that disk as a tupple.
#
# It is the intention to figure out how many copies we have of each file
# identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS
# + on different hosts.
logger.debug("Generate DBM DB with info about independent file copies ...")
complFileListDbm.initKeyPtr()
while (1):
fileKey, fileInfo = complFileListDbm.getNext()
if (not fileKey): break
checkDicKey = ngamsLib.genFileKey(None,
fileInfo[ngamsDbCore.SUM1_FILE_ID],
fileInfo[ngamsDbCore.SUM1_VERSION])
tmpDic = checkDicDbm.get(checkDicKey)
tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo
checkDicDbm.add(checkDicKey, tmpDic)
# Check if there are at least minReqCopies occurrences of the files +
# check that all files are registered (if a Disk ID is specified).
logger.debug("Check for files with less copies than: %s",
str(minReqCopies))
checkDicDbm.initKeyPtr()
while (1):
checkDicKey, tmpDic = checkDicDbm.getNext()
if (not checkDicKey): break
tmpDicKeys = list(tmpDic)
noOfCopies = len(tmpDicKeys)
if (noOfCopies < minReqCopies):
tmpFileInfo = tmpDic[tmpDicKeys[0]]
fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION]
tmpFileObj = ngamsFileInfo.ngamsFileInfo().\
setFileId(fileId).\
setFileVersion(fileVersion).\
setTag("Independent copies: " + str(noOfCopies))
fileKey = ngamsLib.genFileKey(None, fileId, fileVersion)
fileMisCopyDbm.add(str(fileKey), tmpFileObj)
# Remove this file from the Files On Disk DBM - do this only
# if a Disk ID is specified.
if (diskId):
if (diskId in tmpDic):
fileInfo = tmpDic[diskId]
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\
"/" + fileInfo[ngamsDbCore.SUM1_FILENAME])
filename = str(filename)
if (filesOnDiskDicDbm.hasKey(filename)):
filesOnDiskDicDbm.rem(filename)
# Close all DBM objects.
del fileMisCopyDbm
del filesOnDiskDicDbm
del fileListDbm
del fileIdDbm
del complFileListDbm
del checkDicDbm
# The DBM filesOnDiskDicDbmName now contains references to files,
# which are found on the disk but not registered in the DB.
return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def ngamsExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
"""
This DPPI extracts the main header from a FITS file
requested from the ESO Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI controls the call to the printhead module. If the
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=MIDI.2004-02-11T04:16:04.528&
processing=ngasExtractFitsHdrDppi&
processing_pars=header%3D99
The header parameter is optional, if not specified the primary header
is returned.
Valid values for the header parameter are numbers between 0 and 99. If
numbers are specified which are either outside the range or bigger than
the number of headers (including the primary) the primary header will be
returned. Headers are counted from 0 starting with the primary header. 99
is a special value as it returns all headers concatenated in a single file.
If 'xml=vo' is specified headers are returned using a slightly modified
VOTable (XML) format. If 'xml=xfits' is specified headers are returned
using the XFits (XML) format. struct=1 returns the structure of the FITS
file. tsv=1 returns the headers in a tab separated format suitable for
direct ingest into the header repository.
"""
logger.debug("Entering ngasExtractFitsHdrDppi() ...")
statusObj = ngamsDppiStatus.ngamsDppiStatus()
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.\
parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
else:
# default is to extract the primary header
pars = {'header': 0}
logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)
PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])
# initial settings for printhead
xtract = 0
parse = 0
xmlVals = ['xfits', 'vo']
xmlfl = ''
skeyfl = 0
skey = 'END'
show = 0
struct = 0
tsv = 0
check = 0
mergefl = 0
hfl = 0
mode = 1
result = ''
err = ''
ext = 'hdr'
if 'header' in pars:
# extract a certain header: if value == 99 all headers are extracted,
# for any other value that header is extracted. headers are
# counted from 0
hfl = 1
struct = 0
show = pars['header']
try:
head = int(show)
except:
err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
"parameter. Should be int"
if head < 0 or head > 99:
err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
"header parameter."
if 'xml' in pars:
# if this key exists we do a conversion to XFits XML.
struct = 0
if pars['xml'] in xmlVals:
xmlfl = pars['xml']
else:
err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
"parameter. Should be 'vo|xfits': "+ pars['xml']
ext = 'xml'
if 'skey' in pars:
# extract just one keyword. CAUTION: No checking done!
skey = pars['skey'].strip()
skeyfl = 1
keyParts = skey.split()
ext = 'txt'
head = int(head)
if head < 0: head = 0
if ((not re.match('[a-zA-Z]', skey[0]))
or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
or (len(keyParts[0]) > 8)):
err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
"parameter specified. Must be a valid FITS keyword:",\
skey
if 'struct' in pars:
# return only the structure of the FITS file. Value of the
# parameter is ignored
head = -99
struct = 1
ext = 'txt'
if 'tsv' in pars:
# extract header in tsv format. Parameter value is ignored
struct = 1
tsv = 1
ext = 'txt'
head = int(head)
if head < 0: head = 0
if 'check' in pars:
# head structure and calculate the checksum of the data part.
head = -99
struct = 1
check = 1
# printhead supports a list of files, but here we only use one
fils = [filename]
base = os.path.basename(filename)
pos = base.rfind('.fits')
file_id = base[:pos]
if len(set(pars) - PARS) != 0: # detect unsupported parameters
xpars = set(pars) - PARS
err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
"Valid options are:\n" + \
"header=<number> where number is an integer between 0 " +\
"and max(extension)-1 or 99 (for all)\n" +\
"xml=<format>, where format is [vo|xfits]\n" +\
"struct=1\n" +\
"skey=<FITS keyword>, should be a valid keyword, crude " +\
"checking is done\n" + \
"tsv=1\n\n" + \
"combinations are allowed and should be separated by " +\
"a comma character.\n"
err = err % xpars
ext = 'txt'
file_id = 'error'
result = '' # initialize result string
if err != '':
result = err
if result == '':
for f in fils:
cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
check)
logger.debug('Executing command: %s', cmd)
stat, result, stderr = execCmd(cmd)
if stat != 0:
errMsg = "Processing of header for file %s failed:\nstdout:%s\nstderr:%s" % (
filename, utils.b2s(result), utils.b2s(stderr))
raise Exception(errMsg)
resFilename = file_id + "." + ext
try:
# mime-type guessing does not work sometimes, we force it in that case.
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
resFilename)
except:
if ext == 'xml':
mimeType = 'text/xml'
else:
mimeType = 'text/ascii'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
resFilename, '')
statusObj.addResult(resObj)
logger.debug("Leaving ngasExtractFitsHdrDppi() ...")
return statusObj
def _assert_code(self, code, bauth=None, raw_auth=None, cmd='STATUS'):
auth = raw_auth or 'Basic ' + utils.b2s(
base64.b64encode(bauth)) if bauth else None
resp = ngamsHttpUtils.httpGet('127.0.0.1', 8888, cmd, auth=auth)
with contextlib.closing(resp):
self.assertEqual(code, resp.status)
