def _checkContDbSnapshot(testSuiteObj, testCaseNo, dataDirList, waitEmptyCache=True, filterContents=True): """ Function to check the contents of a DB Snapshot file. testSuiteObj: Reference to Test Suite Object (ngamsTestSuite). testCaseNo: Test Case number (integer). dataDirList: List of directories under the NGAS Root Mount Point to check (list). Returns: Void. """ dirPat = testSuiteObj.ngas_path("%s/.db/NgasFiles.bsddb") cacheDirPat = testSuiteObj.ngas_path("%s/.db/cache") refFilePat = "ref/ngamsDbSnapShotTest_test_DbSnapshot_%d_%d.ref" count = 1 startTime = time.time() for dataDir in dataDirList: # Wait till the DB Snapshot Cache Dir is empty. if (waitEmptyCache): cacheDir = cacheDirPat % dataDir startTime = time.time() while (((time.time() - startTime) < 10) and (not os.path.exists(cacheDir))): time.sleep(0.200) startTime = time.time() while ((time.time() - startTime) < 20): tmpDbSnapshot = glob.glob(cacheDir + "/*") if (len(tmpDbSnapshot) == 0): break time.sleep(0.200) complName = dirPat % dataDir refFile = refFilePat % (testCaseNo, count) startTime = time.time() while ((not os.path.exists(complName)) and ((time.time() - startTime) < 10)): time.sleep(0.200) testSuiteObj.assertTrue(os.path.exists(complName), "DB Snapshot missing: " + complName) out = utils.b2s( subprocess.check_output([ sys.executable, '-m', 'ngamsServer.ngamsDumpDbSnapshot', complName ])) if (filterContents): snapshotDump = _parseDbSnapshot(out) else: snapshotDump = out testSuiteObj.assert_ref_file(refFile, snapshotDump, msg="Incorrect contents of DB Snapshot") count += 1
def _authorization_cfg(self, user_specs): uspecs = [(('Name', u[0]), ('Password', utils.b2s(base64.b64encode(u[1]))), ('Commands', u[2])) for u in user_specs] cfg = [('Enable', '1')] for i, u in enumerate(uspecs): cfg += [('User[%d].%s' % (i, name), str(val)) for name, val in u] cfg = [("NgamsCfg.Authorization[1].%s" % name, val) for name, val in cfg] return cfg
def test_UnAuthReq_2(self): """Authorization/authentication is enabled, commands are restricted""" # Start the server with a set of configured users pass1 = os.urandom(16) pass2 = os.urandom(16) pass3 = os.urandom(16) u1 = (('Name', 'test1'), ('Password', utils.b2s(base64.b64encode(pass1))), ('Commands', '*')) u2 = (('Name', 'test2'), ('Password', utils.b2s(base64.b64encode(pass2))), ('Commands', 'STATUS')) u3 = (('Name', 'test3'), ('Password', utils.b2s(base64.b64encode(pass3))), ('Commands', 'DOESNT_EXIST')) cfg = [('Enable', '1')] for i, u in enumerate((u1, u2, u3)): cfg += [('User[%d].%s' % (i, name), str(val)) for name, val in u] cfg = [("NgamsCfg.Authorization[1].%s" % name, val) for name, val in cfg] self.prepExtSrv(cfgProps=cfg) auth1 = b'test1:' + pass1 auth2 = b'test2:' + pass2 auth3 = b'test3:' + pass3 # No authentication self._assert_code(401) # Users 1 and 2 are allowed to STATUS, user 3 isn't self._assert_code(200, bauth=auth1) self._assert_code(200, bauth=auth2) self._assert_code(403, bauth=auth3) # Users 1 and 3 are allowed to send the DOESNT_EXIST command, user 3 isn't # We get 404s here because the command actually doesn't exist, which yields # that HTTP code, and also because HTTP authentication happens before # command processing self._assert_code(404, bauth=auth1, cmd='DOESNT_EXIST') self._assert_code(403, bauth=auth2, cmd='DOESNT_EXIST') self._assert_code(404, bauth=auth3, cmd='DOESNT_EXIST')
def loadVolInfoFile(fname): """Load an NGAS Volume Info file and return it as a dictionary""" with open(fname, "rb") as fo: volInfoBuf = utils.b2s(base64.b64decode(fo.read())) dic = {} for line in volInfoBuf.split("\n"): line = line.strip() if not line or line.startswith("#"): continue par, val = line.split("=") dic[par.strip()] = val.strip() return dic
def test_column_names(self): """Check that column names are correctly bound to data by reading some of the cells and making sure they make sense. If column names are not correclty bound to columns then data might not be convertible and might not make sense""" cfg, _ = self.prepExtSrv() stat = self.assert_query( pars=[['query', 'disks_list'], ['format', 'json']]) results = json.loads(utils.b2s(stat.getData())) self.assertEqual(0, int(results[0]['number_of_files'])) self.assertEqual(cfg.getArchiveName(), results[0]['archive'])
def authorize(cfg, reqPropsObj): """Check if the request is authorized for the authenticated user, if any""" if not cfg.getAuthorize(): logger.debug("Authorization is disabled, continuing anonymously") return # Check if the requested command is excluded from authorization requested_command = reqPropsObj.getCmd().strip() if isCommandExcluded(cfg, requested_command): logger.debug( "Authorization is disabled for the '%s' command, continuing anonymously", requested_command) return # For now only Basic HTTP Authentication is implemented. if not reqPropsObj.getAuthorization(): raise UnauthenticatedError('Unauthorized request received') auth_parts = reqPropsObj.getAuthorization().split(' ') if auth_parts[0] != 'Basic': raise UnauthenticatedError('Invalid authentication scheme: ' + auth_parts[0]) if len(auth_parts) != 2: raise UnauthenticatedError( 'Invalid Basic authentication, missing value') user_pass = base64.b64decode(auth_parts[1]).split(b':') if len(user_pass) < 2: raise UnauthenticatedError( 'Invalid Basic authentication, no password provided') user, password = utils.b2s(user_pass[0]), b':'.join(user_pass[1:]) # Get the user from the configuration. stored_pass = cfg.getAuthUserInfo(user) if not stored_pass: raise UnauthenticatedError("unknown user specified") # Password matches and command is allowed stored_pass = base64.b64decode(six.b(stored_pass)) if password != stored_pass: raise UnauthenticatedError("wrong password for user " + user) if not cmdPermitted(cfg, reqPropsObj, user): raise UnauthorizedError(user) logger.info("Successfully authenticated user %s", user)
def _execCClient(unpackXmlStat = 1, pars = []): """ Execute the NG/AMS C-Client on the shell. cmdLineParsDic: Dictionary with command line parameters (dictionary). pars: Extra parameters for invoking NGAMS C-Client (list). Returns: List with status objects or stdout output from c-client: [<stat obj>, ...] or <stdout c-client> (list|string). """ cmd = ['ngamsCClient'] for opt, val in pars: cmd.append(opt) cmd.append(str(val)) if '-servers' not in (x for x, _ in pars): cmd.append('-host') cmd.append('127.0.0.1') cmd.append('-status') env = os.environ.copy() env['NGAMS_VERBOSE_LEVEL'] = '0' _, out, _ = execCmd(cmd, shell=False) out = utils.b2s(out) if (unpackXmlStat): statObjList = [] xmlStatList = out.split("Command repetition counter:") for xmlStat in xmlStatList: # Clean up the output. xmlStat = xmlStat.strip() if (xmlStat == ""): continue idx = 0 while (xmlStat[idx] != "<"): idx += 1 xmlStat = xmlStat[idx:] # Unpack it. statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat) statObjList.append(statObj) out = statObjList return out
def readDMFStatus(filename): """ Lists file names in long format, giving mode, number of links, owner, group, size in bytes, time of last modification, and, in parentheses before the file name, the DMF state """ cmd = ['dmls', '-l', filename] proc = Popen(cmd, stdout=PIPE, close_fds=True) out, err = proc.communicate() if proc.returncode != 0: raise Exception(out) try: out = utils.b2s(out) status = out.split()[7][1:4] if status in ERROR_STATUS: raise Exception('dmls status error: %s filename: %s' % (status, filename)) return status except IndexError as e: raise Exception('dmls output error')
def loadVolInfoFile(volInfoFile): """ Load the contents of a NGAS Volume Info file and return a dictionary with the parameters. volInfoFile: Path of Volume Info File (string). Returns: Dictionary with the parameters of the Volume Info File (dictionary). """ with open(volInfoFile, "rb") as fo: volInfoBuf = utils.b2s(base64.decodestring(fo.read())) dic = {} for line in volInfoBuf.split("\n"): line = line.strip() if ((line == "") or (line[0] == "#")): continue par, val = line.split("=") dic[par.strip()] = val.strip() return dic
def dumpDbSnapshot(dbSnapshotName, details=0): """ Dump the contents of a DB Snapshot in a raw format. dbSnapshotName: Name of DB Snapshot (string). details: Dump details (integer/0|1). Returns: Void. """ db = bsddb.hashopen(dbSnapshotName, "r") print("\nDumping contents of NG/AMS DB Snapshot: " + dbSnapshotName) try: key, pickleVal = db.first() except: key = None print("") while (key): try: val = cPickle.loads(pickleVal) except Exception as e: tstStr = "No module named" if (str(e).find(tstStr) != -1): modName = str(e).split(tstStr)[-1].strip() exec("import %s" % modName) val = cPickle.loads(pickleVal) else: raise # Try if the object has a dumpBuf() method. try: dumpBuf = val.dumpBuf() except: dumpBuf = None print("%s = %s" % (utils.b2s(key), str(val))) if (details and dumpBuf): print(dumpBuf) try: key, pickleVal = db.next() except: break db.close()
def send_mirrexec_command(self): """ Send MIRREXEC command to the input source_node """ logger.info("Sending MIRREXEC command to %s with (n_threads=%d)", self.target_node, self.num_threads) try: host, port = self.target_node.split(":") pars = { 'n_threads': str(self.num_threads), 'rx_timeout': str(self.rx_timeout), 'iteration': str(self.iteration) } start_time = time.time() # it is important to not let this operation time out. If it times out then the files being fetched will # be eligable for re-fetching even though the spawned threads may still be executing. Chaos ensues. response = ngamsHttpUtils.httpGet(host, int(port), 'MIRREXEC', pars=pars) with contextlib.closing(response): failed = response.status != NGAMS_HTTP_SUCCESS or NGAMS_FAILURE in utils.b2s( response.read()) elapsed_time = time.time() - start_time if failed: logger.error( "MIRREXEC command sent to %s with (n_threads=%d) was handled with status FAILURE " "in %f [s]", self.target_node, self.num_threads, elapsed_time) else: logger.info( "MIRREXEC command sent to %s with (n_threads=%d) was handled with status SUCCESS in %f [s]", self.target_node, self.num_threads, elapsed_time) except Exception: logger.exception("Problems sending MIRREXEC command to %s", self.target_node) return
def checkFileCopiesAndReg(srvObj, minReqCopies, dbFilePat, fileListDbmName=None, diskId=None, ignoreMounted=0): """ The function checks for each file referenced if there are at least 'minReqCopies' copies available somewhere in this NGAS cluster. For the files where this is not the case, an entry is added in a ngasDiskInfo object indicating that this file If an entire disk is analysed (about to be deleted), it is also checked if each file stored on the disk is registered in the DB. Otherwise an NG/AMS File List is returned, containing references to the files not registered. srvObj: Instance of the NG/AMS Server Class (ngamsServer). minReqCopies: Minimum number of copies required (integer). dbFilePat: Filename pattern used to build the DBM containing information about the files (string). fileListDbmName: Name of DBM DB containing explicit references to files to be checked if they can be deleted. The information in this table is pickled lists with the lay-out defined by ngamsDb._ngasFilesCols (string). diskId: Used to refer to all files stored on a disk (string|None). ignoreMounted: Carry out the check also if the disk is not mounted (integer/0|1). Returns: Tuple contaning the filenames of three DBM DBs with the following information: o Files not having the specified number of copies. The contents of this DB are keys (Disk ID + File ID + File Version), pointing to pickled ngamsFileInfo objects. o Files found on the disk but not registered. This contains the complete filenames of files found on the disk, which are not registered. These filenames are the keys of this DBM DB. o Complete list of files referenced in connection with the query. The contents of this DB are keys, which are a simple counter pointing to pickled list containing the information as returned by ngamsDb.getFileSummary1(). (tuple/string). """ T = TRACE() if ((not fileListDbmName) and (not diskId)): errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\ "either a DBM with files to be checked or a Disk ID" logger.warning(errMsg) raise Exception(errMsg) # Create DBMs: # - DB containing information about files having less then the # specified number of copies. fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES") fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1) # - DB that contains information about files stored on the DB, # which are not registered in the NGAS DB. At the end of the function, # this will contain information about files found on the disk but # not registered in the NGAS DB. filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK") filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1) # - DB with information about files referenced by the query. if (not fileListDbmName): locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST") fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1) else: fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0) # - Temporary DBM containing information about all File IDs defined # by the query. fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS") fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1) # - Temporary DBM containing information about all files available in # the system with the File ID/File Version defined by the query. complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST") complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1) # - Temporary DBM that is used to figure out the number of independent # copies of each file concerned by the query. checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC") checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1) # A Disk ID but no file references are given. Retrieve information # about files concerned from the DB. if (diskId): logger.debug("Retrieving information about files on disk with ID: %s", diskId) for f in srvObj.db.getFileSummary1(None, [diskId], [], ignore=0, fileStatus=[]): fileListDbm.addIncKey(f) fileId = f[ngamsDbCore.SUM1_FILE_ID] fileIdDbm.add(str(fileId), "") fileVersion = f[ngamsDbCore.SUM1_VERSION] fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) checkDicDbm.add(fileKey, {}) fileListDbm.sync() fileIdDbm.sync() checkDicDbm.sync() # Get the list of files located on the disk. Later on, remove entries # from this dictionary as the files are parsed, based on their DB info, # further down in this method. # # Key in this dictionary is the complete filename of the file. logger.debug("Get list of files stored on disk ...") tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId) diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ unpackSqlResult(tmpDiskInfo) if ((not ignoreMounted) and (not diskInfoObj.getMounted())): errMsg = "Rejecting request for removing disk with ID: " +\ diskId + " - disk not mounted!" raise Exception(errMsg) if (not ignoreMounted): basePath = os.path.normpath(diskInfoObj.getMountPoint()) pattern = "/*" logger.debug( "Generating list with files on disk with base path: %s", basePath) while (1): tmpFileList = glob.glob(basePath + pattern) if (len(tmpFileList) == 0): break else: for filename in tmpFileList: if (os.path.isfile(filename) and (os.path.basename(filename) != NGAMS_DISK_INFO) and (os.path.basename(filename) != NGAMS_VOLUME_ID_FILE) and (os.path.basename(filename) != NGAMS_VOLUME_INFO_FILE)): filesOnDiskDicDbm.add(str(filename), "") pattern += "/*" # Generate File ID DBM in case a file list DBM is given. if (fileListDbmName): logger.debug("Handling file list DBM given in the function call ...") fileListDbm.initKeyPtr() while (1): key, tmpFileInfo = fileListDbm.getNext() if (not key): break # Update the File ID DBM. fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID] fileIdDbm.add(str(fileId), "") # Update the DBM with references to File ID/Version sets. fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION] fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) checkDicDbm.add(fileKey, {}) fileIdDbm.sync() checkDicDbm.sync() # We need to generate a list with all files available in the system # with the given File ID/File Version. logger.debug("Retrieving information about all files available with the " +\ "File ID/File Version as defined by the query") # Due to the limitation of the size of SQL queries, we have to split up # the SQL query in several sub-queries. The max. length of an SQL query # is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for # the general part of the query, and for each filename we calculate a # length of len(File ID) + 4 as contribution to the SQL query. maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512) queryIds = [] querySize = 0 noOfFileIds = fileIdDbm.getCount() fileIdCount = 0 fileIdDbm.initKeyPtr() fileId = "INIT" while (fileId): fileId, dummy = fileIdDbm.getNext() if (fileId): queryIds.append(utils.b2s(fileId)) fileIdCount += 1 querySize += (len(fileId) + 4) if queryIds and (querySize >= maxQuerySize or fileIdCount == noOfFileIds): for f in srvObj.db.getFileSummary1(fileIds=queryIds, fileStatus=[]): # Take only a sub-result if that File ID + Version # is concerned by the query. tmpFileId = f[ngamsDbCore.SUM1_FILE_ID] tmpFileVersion = f[ngamsDbCore.SUM1_VERSION] tmpFileKey = ngamsLib.genFileKey(None, tmpFileId, tmpFileVersion) if (checkDicDbm.hasKey(tmpFileKey)): complFileListDbm.addIncKey(f) complFileListDbm.sync() queryIds = [] querySize = 0 # Now, go through the files found and order these such that we end up with # a Dictionary with "<File ID>_<File Version>" as keys referring # to a dictionary with the Disk IDs of the disks hosting the files as # keys, and the information for each file on that disk as a tupple. # # It is the intention to figure out how many copies we have of each file # identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS # + on different hosts. logger.debug("Generate DBM DB with info about independent file copies ...") complFileListDbm.initKeyPtr() while (1): fileKey, fileInfo = complFileListDbm.getNext() if (not fileKey): break checkDicKey = ngamsLib.genFileKey(None, fileInfo[ngamsDbCore.SUM1_FILE_ID], fileInfo[ngamsDbCore.SUM1_VERSION]) tmpDic = checkDicDbm.get(checkDicKey) tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo checkDicDbm.add(checkDicKey, tmpDic) # Check if there are at least minReqCopies occurrences of the files + # check that all files are registered (if a Disk ID is specified). logger.debug("Check for files with less copies than: %s", str(minReqCopies)) checkDicDbm.initKeyPtr() while (1): checkDicKey, tmpDic = checkDicDbm.getNext() if (not checkDicKey): break tmpDicKeys = list(tmpDic) noOfCopies = len(tmpDicKeys) if (noOfCopies < minReqCopies): tmpFileInfo = tmpDic[tmpDicKeys[0]] fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID] fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION] tmpFileObj = ngamsFileInfo.ngamsFileInfo().\ setFileId(fileId).\ setFileVersion(fileVersion).\ setTag("Independent copies: " + str(noOfCopies)) fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) fileMisCopyDbm.add(str(fileKey), tmpFileObj) # Remove this file from the Files On Disk DBM - do this only # if a Disk ID is specified. if (diskId): if (diskId in tmpDic): fileInfo = tmpDic[diskId] filename = os.path.\ normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\ "/" + fileInfo[ngamsDbCore.SUM1_FILENAME]) filename = str(filename) if (filesOnDiskDicDbm.hasKey(filename)): filesOnDiskDicDbm.rem(filename) # Close all DBM objects. del fileMisCopyDbm del filesOnDiskDicDbm del fileListDbm del fileIdDbm del complFileListDbm del checkDicDbm # The DBM filesOnDiskDicDbmName now contains references to files, # which are found on the disk but not registered in the DB. return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def ngamsExtractFitsHdrDppi(srvObj, reqPropsObj, filename): """ This DPPI extracts the main header from a FITS file requested from the ESO Archive. srvObj: Reference to instance of the NG/AMS Server class (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). filename: Name of file to process (string). Returns: DPPI return status object (ngamsDppiStatus). Side effect: This DPPI works directly on the archived file, since it is read-only access. SPECIFIC DOCUMENTATION: This DPPI controls the call to the printhead module. If the Example URL (single line): http://ngasdev1:7777/RETRIEVE ?file_id=MIDI.2004-02-11T04:16:04.528& processing=ngasExtractFitsHdrDppi& processing_pars=header%3D99 The header parameter is optional, if not specified the primary header is returned. Valid values for the header parameter are numbers between 0 and 99. If numbers are specified which are either outside the range or bigger than the number of headers (including the primary) the primary header will be returned. Headers are counted from 0 starting with the primary header. 99 is a special value as it returns all headers concatenated in a single file. If 'xml=vo' is specified headers are returned using a slightly modified VOTable (XML) format. If 'xml=xfits' is specified headers are returned using the XFits (XML) format. struct=1 returns the structure of the FITS file. tsv=1 returns the headers in a tab separated format suitable for direct ingest into the header repository. """ logger.debug("Entering ngasExtractFitsHdrDppi() ...") statusObj = ngamsDppiStatus.ngamsDppiStatus() if (reqPropsObj.hasHttpPar("processing_pars")): pars = ngamsPlugInApi.\ parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars")) else: # default is to extract the primary header pars = {'header': 0} logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars) PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check']) # initial settings for printhead xtract = 0 parse = 0 xmlVals = ['xfits', 'vo'] xmlfl = '' skeyfl = 0 skey = 'END' show = 0 struct = 0 tsv = 0 check = 0 mergefl = 0 hfl = 0 mode = 1 result = '' err = '' ext = 'hdr' if 'header' in pars: # extract a certain header: if value == 99 all headers are extracted, # for any other value that header is extracted. headers are # counted from 0 hfl = 1 struct = 0 show = pars['header'] try: head = int(show) except: err = "ngasExtractFitsHdrDppi: Invalid type for header " +\ "parameter. Should be int" if head < 0 or head > 99: err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\ "header parameter." if 'xml' in pars: # if this key exists we do a conversion to XFits XML. struct = 0 if pars['xml'] in xmlVals: xmlfl = pars['xml'] else: err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\ "parameter. Should be 'vo|xfits': "+ pars['xml'] ext = 'xml' if 'skey' in pars: # extract just one keyword. CAUTION: No checking done! skey = pars['skey'].strip() skeyfl = 1 keyParts = skey.split() ext = 'txt' head = int(head) if head < 0: head = 0 if ((not re.match('[a-zA-Z]', skey[0])) or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH') or (len(keyParts[0]) > 8)): err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\ "parameter specified. Must be a valid FITS keyword:",\ skey if 'struct' in pars: # return only the structure of the FITS file. Value of the # parameter is ignored head = -99 struct = 1 ext = 'txt' if 'tsv' in pars: # extract header in tsv format. Parameter value is ignored struct = 1 tsv = 1 ext = 'txt' head = int(head) if head < 0: head = 0 if 'check' in pars: # head structure and calculate the checksum of the data part. head = -99 struct = 1 check = 1 # printhead supports a list of files, but here we only use one fils = [filename] base = os.path.basename(filename) pos = base.rfind('.fits') file_id = base[:pos] if len(set(pars) - PARS) != 0: # detect unsupported parameters xpars = set(pars) - PARS err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \ "Valid options are:\n" + \ "header=<number> where number is an integer between 0 " +\ "and max(extension)-1 or 99 (for all)\n" +\ "xml=<format>, where format is [vo|xfits]\n" +\ "struct=1\n" +\ "skey=<FITS keyword>, should be a valid keyword, crude " +\ "checking is done\n" + \ "tsv=1\n\n" + \ "combinations are allowed and should be separated by " +\ "a comma character.\n" err = err % xpars ext = 'txt' file_id = 'error' result = '' # initialize result string if err != '': result = err if result == '': for f in fils: cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode, check) logger.debug('Executing command: %s', cmd) stat, result, stderr = execCmd(cmd) if stat != 0: errMsg = "Processing of header for file %s failed:\nstdout:%s\nstderr:%s" % ( filename, utils.b2s(result), utils.b2s(stderr)) raise Exception(errMsg) resFilename = file_id + "." + ext try: # mime-type guessing does not work sometimes, we force it in that case. mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), resFilename) except: if ext == 'xml': mimeType = 'text/xml' else: mimeType = 'text/ascii' resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result, resFilename, '') statusObj.addResult(resObj) logger.debug("Leaving ngasExtractFitsHdrDppi() ...") return statusObj
def _assert_code(self, code, bauth=None, raw_auth=None, cmd='STATUS'): auth = raw_auth or 'Basic ' + utils.b2s( base64.b64encode(bauth)) if bauth else None resp = ngamsHttpUtils.httpGet('127.0.0.1', 8888, cmd, auth=auth) with contextlib.closing(resp): self.assertEqual(code, resp.status)