def getFileInfoFromDiskIdFilename(self, diskId, filename):
"""
The method queries the file information for a file referred to by the
Disk ID for the disk hosting it and the filename as stored in the
NGAS DB.
diskId: ID for disk hosting the file (string).
filename: NGAS (relative) filename (string).
Returns: Return ngamsFileInfo object with the information for the
file if found or None if the file was not found
(ngamsFileInfo|None).
"""
T = TRACE()
# Query for the file.
sql = "SELECT %s FROM ngas_files nf WHERE nf.disk_id={0} AND nf.file_name={1}"
sql = sql % (ngamsDbCore.getNgasFilesCols(
self._file_ignore_columnname), )
# Execute the query directly and return the result.
res = self.query2(sql, args=(diskId, filename))
if res:
return ngamsFileInfo.ngamsFileInfo().unpackSqlResult(res[0])
else:
return None
def readHierarchy(self, containerId, includeFiles=False):
"""
Reads an ngamsContainer object from the database
and recursively populates it with its children containers.
:param containerId: string
:return ngamsContainer.ngamsContainer
"""
container = self.read(containerId)
if includeFiles:
# Always get the latest version of the files
# We do this on the software side to avoid any complex SQL query
# that might not work in some engines
sql = "SELECT %s FROM ngas_files nf WHERE container_id = {0} ORDER BY nf.file_id, nf.file_version DESC"
sql = sql % (ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname),)
res = self.query2(sql, args=(containerId,))
prevFileId = None
for r in res:
fileId = r[ngamsDbCore.NGAS_FILES_FILE_ID]
if fileId == prevFileId:
continue
prevFileId = fileId
fileInfo = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r)
container.addFileInfo(fileInfo)
# Check if it contains children
res = self.query2("SELECT container_id FROM ngas_containers WHERE parent_container_id = {0}", args=(containerId,))
for r in res:
container.addContainer(self.readHierarchy(r[0], includeFiles))
return container
def getFileInfoList(self,
diskId,
fileId="",
fileVersion=-1,
ignore=None,
fetch_size=1000):
"""
The function queries a set of files matching the conditions
specified in the input parameters.
diskId: Disk ID of disk hosting the file(s) (string).
fileId: File ID of files to consider. Wildcards can be
used (string).
fileVersion: Version of file(s) to consider. If set to -1 this
is not taken into account (integer).
ignore: If set to 0 or 1, this value of ignore will be
queried for. If set to None, ignore is not
considered (None|0|1).
Returns: Cursor object (<NG/AMS DB Cursor Object API>).
"""
T = TRACE()
cond_sql = collections.OrderedDict()
if fileId:
fileId = re.sub("\*", "%", fileId)
st = "nf.file_id LIKE {}" if '%' in fileId else "nf.file_id = {}"
cond_sql[st] = fileId
if diskId:
cond_sql["nf.disk_id = {}"] = diskId
if fileVersion != -1:
cond_sql["nf.file_version = {}"] = fileVersion
if ignore is not None:
cond_sql["nf.%s = {}" % (self._file_ignore_columnname, )] = ignore
# Create a cursor and perform the query.
sql = [
"SELECT %s FROM ngas_files nf" %
(ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname))
]
if cond_sql:
sql.append(" WHERE ")
sql.append(" AND ".join(cond_sql.keys()))
cursor = self.dbCursor(''.join(sql), args=cond_sql.values())
with cursor:
for x in cursor.fetch(fetch_size):
yield x
def getFileInfoFromFileIdHostId(self,
hostId,
fileId,
fileVersion=1,
diskId=None,
ignore=None):
"""
Return list with information about a certain file referenced
by its File ID. A list is returned with the following elements:
[<Disk ID>, <Filename>, <File ID>, <File Version>, <Format>,
<File Size>, <Uncompressed File Size>, <Compression>,
<Ingestion Date>, <Ignore>, <Checksum>, <Checksum Plug-In>,
<File Status>, <Creation Date>]
hostId: Name of host where the disk is mounted on
which the file is stored (string).
fileId: ID for file to acquire information for (string).
fileVersion: Version of the file to query information
for (integer).
diskId: Used to refer to a specific disk (string).
ignore: If set to 0 or 1, this value of ignore will be
queried for. If set to None, ignore is not
considered (None|0|1).
Returns List with information about file, or [] if
no file(s) was found (list).
"""
T = TRACE()
sql = []
sql.append(("SELECT %s FROM ngas_files nf, ngas_disks nd WHERE "
"nf.file_id={} AND nf.disk_id=nd.disk_id AND "
"nd.host_id={} AND nd.mounted=1 AND nf.file_version={}") \
% ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname))
vals = [fileId, hostId, int(fileVersion)]
if diskId:
sql.append(" AND nd.disk_id={}")
vals.append(diskId)
if ignore:
sql.append(" AND nf.%s={}" % (self._file_ignore_columnname, ))
vals.append(ignore)
res = self.query2(''.join(sql), args=vals)
if not res:
return []
return res[0]
def files_in_host(self, hostId, from_date=None):
"""
Dump the info of the files defined by the parameters. The file info is
dumped into a ngamsDbm DB.
For the parameters check man-page for: ngamsDbBase.getFileSummary1().
Returns: Name of the DBM DB containing the info about the files
(string).
"""
sql, vals = self.buildFileSummary1Query(ngamsDbCore.getNgasFilesCols(
self._file_ignore_columnname),
hostId,
ignore=0,
lowLimIngestDate=from_date,
order=0)
with self.dbCursor(sql, args=vals) as cursor:
for res in cursor.fetch(1000):
yield res
def dumpFileInfoCluster(self,
clusterName,
fileInfoDbmName=None,
useFileKey=False,
count=False):
"""
Dump the info for the files registered in the name space of the
referenced cluster.
Note, all files in the cluster are taken, also the ones marked
as bad or to be ignored.
clusterName: Name of cluster to consider (string).
fileInfoDbmName: Base name of the DBM in which the file info will be
stored. If not given, a name will be generated
automatically (string).
useFileKey: Use a file key (<File ID>_<Version>) as key as
opposed to just an integer key. NOTE: Multiple
occurrences of a given File ID/Version will only
appear once (boolean).
count: When useFileKey == True, if count is True, the
number of ocurrences of each File ID + Version
is counted and an entry added in the DBM:
<File Key>__COUNTER
- pointing to a counter indicating the number of
occurrences. Note, the usage of '__' in the name of
the counter for each file, means it will be skipped
when doing a ngamsDbm.getNext(), scanning through
the contents of the DBM (boolean).
Returns: Final name of the DBM DB containing the info about
the files (string).
"""
T = TRACE()
# Create a temporay File Info DBM.
if (not fileInfoDbmName):
fileInfoDbmName = self.genTmpFile("CLUSTER-FILE-INFO")
rmFile("%s*" % fileInfoDbmName)
# Get list of hosts in the cluster.
clusterHostList = self.getHostIdsFromClusterName(clusterName)
if (clusterHostList == []):
msg = "No hosts registered for cluster with name: %s" % clusterName
raise Exception, msg
clusterHostList = str(clusterHostList).strip()[1:-1]
if (clusterHostList[-1] == ","): clusterHostList = clusterHostList[:-1]
sql = ("SELECT %s FROM ngas_files nf WHERE disk_id IN "
"(SELECT disk_id FROM ngas_disks WHERE "
"host_id IN ({}) OR last_host_id IN ({}))") \
% ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
cursor = self.dbCursor(sql,
args=(clusterHostList, clusterHostList))
fileCount = 1
with cursor:
for fileInfo in cursor.fetch(1000):
if (not useFileKey):
fileInfoDbm.add(str(fileCount), fileInfo)
else:
fileId = fileInfo[ngamsDbCore.NGAS_FILES_FILE_ID]
fileVersion = fileInfo[ngamsDbCore.NGAS_FILES_FILE_VER]
fileKey = ngamsLib.genFileKey(None, fileId,
fileVersion)
if (count):
countKey = "%s__COUNTER" % fileKey
if (not fileInfoDbm.hasKey(countKey)):
fileInfoDbm.add(countKey, 0)
fileInfoDbm.add(countKey,
(fileInfoDbm.get(countKey) + 1))
fileInfoDbm.add(fileKey, fileInfo)
fileCount += 1
fileInfoDbm.sync()
del fileInfoDbm
except Exception, e:
rmFile(fileInfoDbmName)
msg = "dumpFileInfoCluster(): Failed in dumping file info. " +\
"Error: %s" % str(e)
raise Exception, msg
def getFileInfoFromFileId(self,
fileId,
fileVersion=-1,
diskId=None,
ignore=None,
dbCursor=1,
order=1):
"""
The method queries the file information for the files with the given
File ID and returns the information found in a list containing
sub-lists each with a list with the information for the file from the
ngas_files table, host ID and mount point. The following rules are
applied when determining which files to return:
o All files are considered, also files which are Offline.
o Files marked to be ignored are ignored.
o Latest version - first priority.
It is possible to indicate if files marked as being 'bad' in the
DB should be taken into account with the 'ignoreBadFiles' flag.
If a specific File Version is specified only that will be
taken into account.
The data can be retrieved via the DB Cursor returned by this object.
The format of each sub-result is:
[<see getFileInfoFromFileIdHostId()>, <host ID>, <mnt pt>]
fileId: File ID for file to be retrieved (string).
fileVersion: If a File Version is specified only information
for files with that version number and File ID
are taken into account. The version must be a
number in the range [1; oo[ (integer).
diskId: ID of disk where file is residing. If specified
to None (or empty string) the Disk ID is not taken
into account (string).
ignore: If set to 0 or 1, this value of ignore will be
queried for. If set to None, ignore is not
considered (None|0|1).
dbCursor: If set to 1, a DB cursor is returned from which
the files can be retrieved. Otherwise the result
is queried and returned in a list (0|1/integer).
order: If set to 0, the list of matching file information
will not be order according to the file version
(integer/0|1).
Returns: Cursor object or list with results
(<NG/AMS DB Cursor Object API>|list).
"""
T = TRACE()
try:
int(fileVersion)
except:
raise Exception, "Illegal value for File Version specified: " +\
str(fileVersion)
sql = []
vals = []
# Query for files being online.
sql.append(("SELECT %s, nd.host_id, "
"nd.mount_point FROM ngas_files nf, ngas_disks nd, "
"ngas_hosts nh WHERE nh.host_id=nd.host_id AND "
"nf.disk_id=nd.disk_id") %
ngamsDbCore.getNgasFilesCols(self._file_ignore_columnname))
if ignore:
sql.append(" AND nf.%s={}" % (self._file_ignore_columnname, ))
vals.append(ignore)
# File ID specified.
if fileId:
sql.append(" AND nf.file_id={}")
vals.append(fileId)
# Do we want a specific File Version?
if fileVersion != -1:
sql.append(" AND nf.file_version={}")
vals.append(int(fileVersion))
# Is a special disk referred?
if diskId:
sql.append(" AND nf.disk_id={}")
vals.append(diskId)
# Order the files according to the version.
if order:
sql.append(" ORDER BY nf.file_version desc, nd.disk_id desc")
if dbCursor:
return self.dbCursor(''.join(sql), args=vals)
res = self.query2(''.join(sql), args=vals)
if not res:
return []
return res
