def checkTarball(filename):
"""
Check that the tarball is correct
filename: Name of tarball file (string).
Returns: Void.
"""
err = 0
tmpFilename = filename.replace(":", "_") + "_tmp"
try:
execCmd("ln -s " + filename + " " + tmpFilename)
stat, out, _ = execCmd("tar -tf " + tmpFilename)
rmFile(tmpFilename)
if (stat != 0):
errMsg = str(out).replace("\n", " --- ")
err = 1
except Exception as e:
rmFile(tmpFilename)
errMsg = str(e)
err = 1
if (err):
errMsg = "Error checking tarball: " + errMsg
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[filename, "ngasTarBallPlugIn", errMsg])
raise Exception(errMsg)
def ngamsSuspensionPlugIn(srvObj):
"""
Suspension Plug-In to suspend an NGAS host.
srvObj: Reference to instance of the NG/AMS Server (ngamsServer).
Returns: Void.
"""
execCmd("sudo /sbin/shutdown -h now")
def run(pars, srvObj, filename):
user = pars['user']
host = pars['host']
tgt_dir = pars['tgt_dir']
tgt_filename = os.path.join(tgt_dir, os.path.basename(filename))
tgt = '%s@%s:%s' % (user, host, tgt_filename)
execCmd(['scp', filename, tgt], timeOut=300, shell=False)
logger.info("Successfully scp'd logfile %s to %s", filename, tgt)
def ngamsWakeUpPlugIn(srvObj, hostId):
"""
Wake-Up Plug-In to wake up a suspended NGAS host.
srvObj: Reference to instance of the NG/AMS Server (ngamsServer).
hostId: Name of NGAS host to be woken up (string).
Returns: Void.
"""
hostDic = ngamsHighLevelLib.\
getHostInfoFromHostIds(srvObj.getDb(), [hostId])
if hostId not in hostDic:
errMsg = "ngamsWakeUpPlugIn: Could not wake up host: " + hostId +\
" - host not defined in NGAS DB."
raise Exception(errMsg)
networkDevs = srvObj.getCfg().getWakeUpPlugInPars()
cmdFormat = "sudo /sbin/ether-wake -i %s -b " +\
hostDic[hostId].getMacAddress()
logger.debug("Waking up suspended host: %s", hostId)
for dev in networkDevs.split(","):
cmd = cmdFormat % dev
logger.debug("Broadcasting wake-up package - command: %s", cmd)
stat, out, _ = execCmd(cmd)
if (stat != 0):
format = "ngamsWakeUpPlugIn: Problem waking up host: %s " +\
". Error: %s."
errMsg = format % (hostId, str(out).replace("\n", " "))
raise Exception(errMsg)
def mountToMountpoint(devName, mntPt, readOnly, fstype="reiserfs"):
"""
Method mounts a device <devName> to a mount point <mntPt>. The optional
parameter fstype can be used to specify a filesystem type different than
reiserfs. To mount a device which is in the /etc/fstab use the
following command:
mountToMountpoint("", <mntPt>, fstype="")
Synopsis: mountToMountpoint(devName, mntPt[, fstype = <fstype>])
devName: Device name (string).
mntPt: Mount point (string).
readOnly: 0 if not read-only, 1 if read-only (integer).
fstype: File system type (string).
Returns: Void.
"""
command = ["sudo", "mount"]
if (readOnly):
command.append('-r')
command += [devName, mntPt]
logger.debug("Command to mount disk: %s", subprocess.list2cmdline(command))
# if (getMountedDev(mntPt)): # already mounted
# warnMsg = "Device " + getMountedDev(mntPt) + ' already mounted ' + \
# 'to specified mountpoint: ' + mntPt
# info(1,warnMsg)
# else:
# if not os.path.exists(mntPt):
# try:
# posix.mkdir(mntPt)
# except exceptions.OSError,e:
# errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e)
# error(errMsg)
# raise Exception, errMsg
# stat, out = commands.getstatusoutput(command)
# if (stat != 0):
# errMsg = "Failed mounting device " + getMountedDev(mntPt) + \
# ':' + out
# raise Exception, errMsg
if not os.path.exists(mntPt):
try:
os.mkdir(mntPt)
except OSError as e:
errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e)
logger.exception(errMsg)
raise
stat, out, _ = execCmd(command)
if stat != 0 or b'already mounted' in out:
errMsg = "Failed mounting device. Device/mount point: %s/%s" %\
(devName, mntPt)
raise Exception(errMsg)
def handle_event(evt):
if evt.file_version != 1:
logger.info('File %s arrived with version != 1 (=%d), skipping',
evt.file_id, evt.file_version)
return
if evt.file_id not in self.file_ids:
logger.info(
'File %s not in list of files to be source-extracted, skipping',
evt._file_id)
return
cmd = [x.replace('%s', evt.file_id) for x in self.command]
logger.info('Executing command to trigger source extraction: %s',
subprocess.list2cmdline(cmd))
ngamsCore.execCmd(cmd)
def rmMod(module):
"""
Wrapper around the rmmod program.
NOTE: This is Linux specific and won't work on any other system.
Synopsis: rmMod(<module>)
module: Module name (string).
Returns: 0 upon success or if module is not loaded (int).
"""
stat = checkModule(module)
if (stat[1]):
execCmd("/sbin/rmmod " + module)
else:
return 0
return 0
def exportCont(contId):
"""
Export all units found on the controller. The function does not complain
if errors occur.
contId: ID (number) of the controller (integer).
Returns: Void.
"""
return # We don't want this to be executed ...
# Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC
# ------------------------------------------------------------------------------
# u0 RAID-5 OK - 256K 1862.59 ON OFF OFF
# u1 RAID-5 OK - 256K 1490.07 ON OFF OFF
# u2 SPARE OK - - 372.603 - OFF -
# Port Status Unit Size Blocks Serial
# ---------------------------------------------------------------
# p0 OK u0 372.61 GB 781422768 KRFS02RAGWWHKC
# p1 OK u0 372.61 GB 781422768 KRFT06RAGW9ZAC
# p2 OK u0 372.61 GB 781422768 KRFS26RAGWB7ZC
# ...
cmd = "sudo /usr/local/sbin/tw_cli info c%s" % str(contId)
stat, out, _ = execCmd(cmd)
# Just look for lines starting with "u".
unitList = []
for line in out.split("\n"):
line = line.strip()
if (line == ""): continue
if (line[0] == "u"): unitList.append(line.split(" ")[0])
for unit in unitList:
cmd = "sudo /usr/local/sbin/tw_cli /c%d/%s export quiet" %\
(int(contId), unit)
logger.debug("Invoking command to export 3ware unit: %s ...", cmd)
stat, out, _ = execCmd(cmd)
logger.debug("Result of command: %s to export 3ware unit: %d", cmd,
stat)
def rescanCont(contId):
"""
Rescan a 3ware controller.
contId: ID (number) of the controller (integer).
Returns: Void.
"""
return # Don't want to execute this
cmd = "sudo /usr/local/sbin/tw_cli /c%d rescan" % (int(contId))
logger.debug("Invoking command to rescan 3ware unit: %s ...", cmd)
stat, out, _ = execCmd(cmd)
logger.debug("Result of command: %s to rescan 3ware unit: %d", cmd, stat)
def checkModule(module):
"""
Check whether module <module> exists and is loaded. Method returns
a tuple, <mtup>, where mtab[0] reports the existence of a module, and
mtab[1] reports the load status of a module.
NOTE: This is Linux specific and won't work on any other system
Synopsis: checkModule(<module>)
module: Module name (string).
Returns: 2-element tuple (tuple).
"""
mex, ml = (-1, -1) # default return code
if type(module) != type(""):
errMsg = "Parameter module passed to " + __name__ + "." + \
"checkModule has invalid type (should be string)"
raise Exception(errMsg)
# command = "/sbin/modprobe -l | /bin/grep \"/" + module + ".o$\""
command = "find /lib/modules -name " + module + ".*"
stat, out, _ = execCmd(command)
if stat > 0 or not out.strip():
mex, ml = (0, 0) # there is no module like this
else:
mex = 1 # module exists
stat, out, _ = execCmd("/sbin/lsmod | /bin/grep " + \
"\"^" + module + "\"")
if out:
ml = 1 # module is loaded
else:
ml = 0 # module not loaded
mtup = (mex, ml)
return mtup
def umountMountpoint(mntPt):
"""
Method unmounts a device mounted on <mntPt>.
Synopsis: unmountMountpoint(mntPt)
mntPoint: Mount point (string).
Returns: Void.
"""
command = ["sudo", "umount", mntPt]
stat, out, _ = execCmd(command)
if stat != 0 or b'not mounted' in out:
errMsg = "Failed unmounting. Mount point: %s" % mntPt
raise Exception(errMsg)
def _execCClient(unpackXmlStat = 1,
pars = []):
"""
Execute the NG/AMS C-Client on the shell.
cmdLineParsDic: Dictionary with command line parameters (dictionary).
pars: Extra parameters for invoking NGAMS C-Client (list).
Returns: List with status objects or stdout output from c-client:
[<stat obj>, ...] or <stdout c-client> (list|string).
"""
cmd = ['ngamsCClient']
for opt, val in pars:
cmd.append(opt)
cmd.append(str(val))
if '-servers' not in (x for x, _ in pars):
cmd.append('-host')
cmd.append('127.0.0.1')
cmd.append('-status')
env = os.environ.copy()
env['NGAMS_VERBOSE_LEVEL'] = '0'
_, out, _ = execCmd(cmd, shell=False)
out = utils.b2s(out)
if (unpackXmlStat):
statObjList = []
xmlStatList = out.split("Command repetition counter:")
for xmlStat in xmlStatList:
# Clean up the output.
xmlStat = xmlStat.strip()
if (xmlStat == ""): continue
idx = 0
while (xmlStat[idx] != "<"): idx += 1
xmlStat = xmlStat[idx:]
# Unpack it.
statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat)
statObjList.append(statObj)
out = statObjList
return out
def getContInfo(contList):
"""
Get the dump about each controller, concatenate this and return it.
contList: List with controllers in the system (list/integer).
Returns: Accumulated ASCII output of info about controllers (string).
"""
out = ""
for contId in contList:
cmd = "sudo /usr/local/sbin/tw_cli info c%s" % str(contId)
logger.debug("Executing 3ware client tool: %s", cmd)
stat, outTmp, _ = execCmd(cmd)
logger.debug("Executed 3ware client tool. Status: %d", stat)
if (stat):
raise Exception("Error invoking 3ware Command Line Tool: " +\
str(out))
else:
out += outTmp
out += "\n"
return out
def ngamsTileCompress(filename):
"""
Tile compress the referenced file.
filename: Filename (string).
Returns: Void.
"""
tmpFilename = filename + ".tmp"
try:
comprCmd = "imcopy %s '%s[compress]'" % (filename, tmpFilename)
logger.debug("Command to tile compress file: %s", comprCmd)
stat, out, _ = execCmd(comprCmd)
if (stat != 0):
msg = "Error compressing file: %s. Error: %s" %\
(filename, stat.replace("\n", " "))
raise Exception(msg)
mvFile(tmpFilename, filename)
logger.debug("Successfully tile compressed file: %s", filename)
except:
rmFile(tmpFilename)
raise
def getControllers():
"""
Query the controllers available, and return this in a list.
Returns: List with available 3ware controllers (list).
"""
cmd = "sudo /usr/local/sbin/tw_cli info"
stat, out, _ = execCmd(cmd)
if (stat):
raise Exception("Error invoking 3ware Command Line Tool: " + str(out))
contList = []
for line in out.split("\n"):
line = line.strip()
if (line):
if (line.find("Controller") == 0):
# "Controller 1: 8506-4LP (4)"
contNo = line.split(" ")[1].split(":")[0]
contList.append(contNo)
elif (line[0] == "c"):
contNo = int(line.split(" ")[0][1:])
contList.append(contNo)
contList.sort()
return contList
def ngasExtractFitsHdrDppi(srvObj, reqPropsObj, filename):
"""
This DPPI extracts the main header from a FITS file
requested from the ESO Archive.
srvObj: Reference to instance of the NG/AMS Server
class (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
filename: Name of file to process (string).
Returns: DPPI return status object (ngamsDppiStatus).
Side effect: This DPPI works directly on the archived file, since it is
read-only access.
SPECIFIC DOCUMENTATION:
This DPPI controls the call to the printhead module. If the
Example URL (single line):
http://ngasdev1:7777/RETRIEVE
?file_id=MIDI.2004-02-11T04:16:04.528&
processing=ngasExtractFitsHdrDppi&
processing_pars=header%3D99
The header parameter is optional, if not specified the primary header
is returned.
Valid values for the header parameter are numbers between 0 and 99. If
numbers are specified which are either outside the range or bigger than
the number of headers (including the primary) the primary header will be
returned. Headers are counted from 0 starting with the primary header. 99
is a special value as it returns all headers concatenated in a single file.
If 'xml=vo' is specified headers are returned using a slightly modified
VOTable (XML) format. If 'xml=xfits' is specified headers are returned
using the XFits (XML) format. struct=1 returns the structure of the FITS
file. tsv=1 returns the headers in a tab separated format suitable for
direct ingest into the header repository.
"""
logger.debug("Entering ngasExtractFitsHdrDppi() ...")
statusObj = ngamsDppiStatus.ngamsDppiStatus()
if (reqPropsObj.hasHttpPar("processing_pars")):
pars = ngamsPlugInApi.\
parseRawPlugInPars(reqPropsObj.getHttpPar("processing_pars"))
else:
# default is to extract the primary header
pars = {'header': 0}
logger.debug("ngasExtractFitsHdrDppi: %s %r", filename, pars)
PARS = set(['header', 'xml', 'skey', 'struct', 'tsv', 'check'])
# initial settings for printhead
xtract = 0
parse = 0
xmlVals = ['xfits', 'vo']
xmlfl = ''
skeyfl = 0
skey = 'END'
show = 0
struct = 0
tsv = 0
check = 0
mergefl = 0
hfl = 0
mode = 1
result = ''
err = ''
ext = 'hdr'
if pars.has_key('header'):
# extract a certain header: if value == 99 all headers are extracted,
# for any other value that header is extracted. headers are
# counted from 0
hfl = 1
struct = 0
show = pars['header']
try:
head = int(show)
except:
err = "ngasExtractFitsHdrDppi: Invalid type for header " +\
"parameter. Should be int"
if head < 0 or head > 99:
err = "ngasExtractFitsHdrDppi: Invalid value specified for " +\
"header parameter."
if pars.has_key('xml'):
# if this key exists we do a conversion to XFits XML.
struct = 0
if pars['xml'] in xmlVals:
xmlfl = pars['xml']
else:
err = "ngasExtractFitsHdrDppi: Invalid value for xml " +\
"parameter. Should be 'vo|xfits': "+ pars['xml']
ext = 'xml'
if pars.has_key('skey'):
# extract just one keyword. CAUTION: No checking done!
skey = pars['skey'].strip()
skeyfl = 1
keyParts = skey.split()
ext = 'txt'
head = int(head)
if head < 0: head = 0
if ((not re.match('[a-zA-Z]', skey[0]))
or (len(keyParts) > 1 and keyParts[0] != 'HIERARCH')
or (len(keyParts[0]) > 8)):
err = "ngasExtractFitsHdrDppi: Invalid value for skey " +\
"parameter specified. Must be a valid FITS keyword:",\
skey
if pars.has_key('struct'):
# return only the structure of the FITS file. Value of the
# parameter is ignored
head = -99
struct = 1
ext = 'txt'
if pars.has_key('tsv'):
# extract header in tsv format. Parameter value is ignored
struct = 1
tsv = 1
ext = 'txt'
head = int(head)
if head < 0: head = 0
if pars.has_key('check'):
# head structure and calculate the checksum of the data part.
head = -99
struct = 1
check = 1
# printhead supports a list of files, but here we only use one
fils = [filename]
base = os.path.basename(filename)
pos = base.rfind('.fits')
file_id = base[:pos]
if len(set(pars) - PARS) != 0: # detect unsupported parameters
xpars = set(pars) - PARS
err = "ngasExtractFitsHdrDppi: Unsupported option(s): %s\n" + \
"Valid options are:\n" + \
"header=<number> where number is an integer between 0 " +\
"and max(extension)-1 or 99 (for all)\n" +\
"xml=<format>, where format is [vo|xfits]\n" +\
"struct=1\n" +\
"skey=<FITS keyword>, should be a valid keyword, crude " +\
"checking is done\n" + \
"tsv=1\n\n" + \
"combinations are allowed and should be separated by " +\
"a comma character.\n"
err = err % xpars
ext = 'txt'
file_id = 'error'
result = '' # initialize result string
if err != '':
result = err
if result == '':
for f in fils:
cmd = constructCommand(f, head, struct, skey, tsv, xmlfl, mode,
check)
logger.debug('Executing command: %s', cmd)
stat, result, _ = execCmd(cmd)
if stat != 0:
errMsg = "Processing of header for file %s failed: %s" % (
filename, result)
raise Exception(errMsg)
resFilename = file_id + "." + ext
try:
# mime-type guessing does not work sometimes, we force it in that case.
mimeType = ngamsPlugInApi.determineMimeType(srvObj.getCfg(),
resFilename)
except:
if ext == 'xml':
mimeType = 'text/xml'
else:
mimeType = 'text/ascii'
resObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_DATA, mimeType, result,
resFilename, '')
statusObj.addResult(resObj)
logger.debug("Leaving ngasExtractFitsHdrDppi() ...")
return statusObj
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def runAllTests(notifEmail=None, skip=None):
"""
Run all tests in ngasUtils/test.
notifEmail: List of email recipients that should be notified about the
test results (list/email addresses).
skip: Test Suites or Test Cases to skip. Comma separated list
(string).
Returns: Void.
"""
skipDic = {}
if (skip):
for test in skip.split(","):
skipDic[test] = 1
testModList = getTestList()
startTime = time.time()
failModDic = {}
noOfTests = len(testModList)
testCount = 0
format = "Running Test Suite: %-32s %-8s"
line = "\nNGAS UTILITIES FUNCTIONAL TESTS - TEST REPORT\n"
print line
testRep = line + "\n"
line = "Date: %s" % PccUtTime.TimeStamp().getTimeStamp()
print line
testRep += line + "\n"
line = "Host: %s" % getHostName()
print line
testRep += line + "\n"
line = "NG/AMS Version: %s\n" % getNgamsVersion()
print line
testRep += line + "\n"
for mod in testModList:
if (skipDic.has_key(mod)): continue
testCount += 1
line = format % (mod, ("(#%d/%d)" % (testCount, noOfTests)))
testRep += line
sys.stdout.write(line)
sys.stdout.flush()
suiteStartTime = time.time()
stat, stdout, stderr = ngamsCore.execCmd("python " + mod + ".py",
NGAMS_TEST_MAX_TS_TIME)
testTime = (time.time() - suiteStartTime)
if (testTime >= NGAMS_TEST_MAX_TS_TIME):
failModDic[mod] = "TIME-OUT"
stat = " - %-5.1fs - TIME-OUT!!\n" % testTime
elif (stdout.find("FAILED") != -1):
failModDic[mod] = stdout + " --- " + stderr
stat = " - %-5.1fs - FAILURE!!\n" % testTime
else:
stat = " - %-5.1fs - SUCCESS\n" % testTime
sys.stdout.write(stat)
sys.stdout.flush()
testRep += stat
execTime = (time.time() - startTime)
line = "\n\nExecuted %d Test Suites in %.3f seconds\n\n" %\
(len(testModList), execTime)
print line
testRep += line
if (failModDic):
line = 80 * "="
print line
testRep += line
line = "\n\nFAILED TEST SUITES:\n\n"
print line
testRep += line
failMods = failModDic.keys()
failMods.sort()
for failMod in failMods:
line = "\n%s:\n%s\n" % (failMod, failModDic[failMod])
print line
testRep += line + "\n"
line = 80 * "-"
print line
testRep += line
line = 80 * "="
print line
testRep += line
# Send out email with test report if requested.
if (notifEmail):
notifEmail = notifEmail.split(",")
par = "NgamsCfg.Server[1].MountRootDirectory"
ngamsCfgObj = ngamsConfig.ngamsConfig().storeVal(par, "/NGAS")
ngamsHighLevelLib.sendEmail(ngamsCfgObj, "localhost",
"NGAS UTILITIES FUNCTIONAL TESTS REPORT",
notifEmail, "ngas@%s" %\
ngamsLib.getCompleteHostName(), testRep)
def parseGen2Controller(rootMtPt, contList, devStartIdx, ngasStartIdx,
contOut):
"""
The output of the 2nd generation 3ware Command Line Utility used for NGAS
look like this:
Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC
---------------------------------------------------------------------------
Port Status Unit Size Blocks Serial
---------------------------------------------------------------
p0 NOT-PRESENT - - - -
p1 OK - 372.61 GB 781422768 KRFS06RAGGXTAC
p2 NOT-PRESENT - - - -
p3 OK - 372.61 GB 781422768 KRFS06RAGBND5C
p4 OK - 372.61 GB 781422768 KRFS06RAGGWD7C
p5 OK - 372.61 GB 781422768 KRFS06RAGGXTAC
p6 OK - 372.61 GB 781422768 KRFS06RAGGXUWC
p7 OK - 372.61 GB 781422768 KRFS06RAGBND5C
Unit UnitType Status %Cmpl Stripe Size(GB) Cache AVerify IgnECC
---------------------------------------------------------------------------
u0 RAID-1 OK - - 65.1826 ON OFF OFF
u1 RAID-1 OK - - 372.519 ON OFF OFF
Port Status Unit Size Blocks Serial
---------------------------------------------------------------
p0 OK u0 69.25 GB 145226112 WD-WMAKE1317691
p1 OK u0 69.25 GB 145226112 WD-WMAKE1317543
p2 OK u1 372.61 GB 781422768 KRFS06RAGBNB1C
p3 OK u1 372.61 GB 781422768 KRFS06RAGBGP0C
rootMtPt: Root mount point (string).
contList: List with controllers in the system (list/integer).
devStartIdx: Start index of disk array (string/a|b|c...).
ngasStartIdx: Startindex of first disk to be used by NGAS (string/a|b|c...)
contOut: Output from 3ware controller (string).
Returns: Dictionary with info about the disks (dictionary).
"""
# Parse the output from the 3ware Command Line Tool.
diskInfoDic = {}
slotCount = 0 # Not used for the moment.
contCount = 0 # Controller counter.
tmpPorts = 0 # Number of ports current controller.
portsSum = 0 # Total sum of ports parsed.
devIdx = ord(devStartIdx)
if ngasStartIdx < devStartIdx: ngasStartIdx = devStartIdx
idx = 0
bufLines = contOut.split("\n")
bufLen = len(bufLines)
while (idx < bufLen):
line = bufLines[idx].strip()
logger.debug("Parsing 3ware status line: %s", line)
# Next controller?
if ((line.find("Unit ") != -1) and (line.find("UnitType ") != -1)):
unitTypeDic = {}
contNo = contList[contCount]
contCount += 1
portsSum += tmpPorts
tmpPorts = 0
# Look for 1st unit - check if a JBOD or RAID unit.
raid = 0
while (idx < bufLen):
idx += 1
line = bufLines[idx].strip()
if (len(line)):
if ((line.find("Port") != -1)
and (line.find("Status") != -1)):
break
elif (line[0].strip() == "u"):
cfg = filter(None, line.split(" "))
if (cfg[1].find("RAID") != -1):
unitTypeDic[cfg[0]] = "RAID"
else:
unitTypeDic[cfg[0]] = "JBOD"
# Get the info for each JBOD disk. Parse until the start of the
# next controller status or till the end of the status output.
idx += 1
while (idx < bufLen):
line = bufLines[idx].strip()
logger.debug("Parsing 3ware status line: %s", line)
if (line == ""):
idx += 1
continue
if ((line.find("Unit ") != -1)
and (line.find("UnitType ") != -1)):
break # Break to outer loop
# Take the line if it contains disk info.
if (line[0] == "p"):
tmpPorts += 1
diskInfo = filter(None, line.split(" "))
if ((diskInfo[1] != "NOT-PRESENT")
and (unitTypeDic[diskInfo[2]] == "JBOD")):
# A disk is found, get the info.
portNo = int(diskInfo[0][1:])
slotId = str((((contCount - 1) > 0) * portsSum) +\
(portNo + 1))
# Get status, model ID, serial number and capacity.
cmd = "sudo /usr/local/sbin/tw_cli info c%d p%d " +\
"status model serial capacity"
cmd = cmd % (int(contNo), int(portNo))
stat, out, _ = execCmd(cmd)
outLines = out.split("\n")
status = filter(None, outLines[0].split(" "))[3]
model = filter(None, outLines[1].split(" "))[3]
serialNo = filter(None, outLines[2].split(" "))[3]
capGb = int(float(filter(None, outLines[3].\
split(" "))[3]) + 0.5)
diskType = "MAGNETIC DISK/ATA"
if (model.find("HDS") == 0):
manufact = "Hitachi Data Systems"
elif (model.find("WDC") == 0):
manufact = "Western Digital Corporation"
else:
manufact = "UNKNOWN"
diskId = model + "-" + serialNo
mtPt = rootMtPt + "/volume" + slotId
devName = "/dev/sd" + chr(devIdx +\
len(diskInfoDic)) + "1"
diskInfoDic[str(slotId)] = ngamsPhysDiskInfo.\
ngamsPhysDiskInfo().\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus(status).\
setCapacityGb(capGb).\
setModel(model).\
setSerialNo(serialNo).\
setType(diskType).\
setManufacturer(manufact).\
setDiskId(diskId).\
setDeviceName(devName)
idx += 1
idx -= 1 # Decrease count to avoid index exceeding buffer length.
idx += 1 # Main loop buffer index
# AWI: added to fix ATF problem [AW]
diskInfoDicUsed = {}
for di in diskInfoDic:
if diskInfoDic[di].getDeviceName() >= '/dev/sd' + ngasStartIdx + '1':
diskInfoDicUsed.update({di: diskInfoDic[di]})
# AWI
return diskInfoDicUsed
def parseHtmlInfo(url, rootMtPt, slotIds=["*"]):
"""
Query the information about the disks as an HTML page, and parse the
page and return the information in the format as defined for the
NG/AMS Disk Dictionary.
url: URL from where to query the disk status (string).
rootMtPt: Root mount point to be used (string)
slotIds: List containing the Slot IDs to take into account. If
given as '*' all slot are considered (list/string).
Returns: Dictionary containing objects which are instances of
the class ngamsPhysDiskInfo (dictionary).
"""
# Check if the 3ware WEB server can be accessed.
cmd = "wget -T 2 -t 1 %s" % url
stat, _, _ = execCmd(cmd)
if (stat):
logger.warning("Problem contacting local 3ware WEB server via URL: %s",
url)
return {}
fd = urllib.urlopen(url)
#fd = open(url)
buf = fd.read()
fd.close()
lines = buf.split("\n")
idx = 0
length = len(lines)
takeAllSlots = 0
if (len(slotIds)):
if (slotIds[0] == '*'):
takeAllSlots = 1
diskInfoDic = {}
while (idx < length):
if "tech_drive_header\">Port" in lines[idx]:
# We expect the following output:
#
# <tr><td class="tech_drive_header">Port 0</td></tr>
# <tr><td class="tech_drive_info">
# <table>
# <tr><td class="tech_unit_field">Status:</td>\
# <td class="tech_unit_value">\
# <font color="#3333FF">OK</font></td></tr>
# <tr><td class="tech_unit_field">Capacity:\
# </td><td class="tech_unit_value">\
# 38.34 GB (80418240 blocks)</td></tr>
# <tr><td class="tech_unit_field">\
# Model:</td><td class="tech_unit_value">\
# IBM-DTLA-305040 </td></tr>
# <tr><td class="tech_unit_field">Serial number:\
# </td><td class="tech_unit_value"> YJ0YJ070913</td>\
# </tr>
# <tr><td class="tech_unit_field">Unit number:</td>\
# <td class="tech_unit_value">0</td></tr>
# <li>Physical drive (port) 7
# <ul>
# <li>Status: <font color="#3333FF">OK</font>
# <li>Capacity: 41.17 GB (80418240 blocks)
# <li>Model: IBM-DTLA-305040
# <li>Serial number: YJ0YJ070913
# <li>Unit number: 7
portNo = int(lines[idx].split(" ")[2].split("<")[0])
# Check if this slot should be taken into account, i.e., if the
# current Slot ID is in the list given as input to the function.
slotId = str(int(portNo) + 1)
if ((not takeAllSlots) and slotId not in slotIds):
idx = idx + 1
continue
idx = idx + 3
status = lines[idx].split(">")[5].split("<")[0]
idx = idx + 1
capacityGb = float(lines[idx].split(">")[4].split(" ")[0])
capacityBlocks = int(
lines[idx].split(">")[4].split("(")[1].split(" ")[0])
idx = idx + 1
model = lines[idx].split(">")[4].split(" ")[0]
diskType = "MAGNETIC DISK/ATA"
manufacturer = model.split("-")[0]
if (manufacturer[0:2] == "IC"): manufacturer = "IBM"
idx = idx + 1
serialNo = lines[idx].split(">")[4].split("<")[0].strip()
idx = idx + 1
unitNo = int(lines[idx].split(">")[4].split("<")[0])
diskId = model + "-" + serialNo
mtPt = rootMtPt + "/data" + str(slotId)
deviceName = "/dev/sd" + chr(97 + len(diskInfoDic)) + '1'
diskInfoDic[str(slotId)] = ngamsPhysDiskInfo.ngamsPhysDiskInfo().\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus(status).\
setCapacityGb(capacityGb).\
setModel(model).\
setSerialNo(serialNo).\
setType(diskType).\
setManufacturer(manufacturer).\
setDiskId(diskId).\
setDeviceName(deviceName)
idx = idx + 1
return diskInfoDic
