def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg):
"""
Makes a physical file from a virtual one
Creates a temp file with results from grep operation/cating of files
Returns: [machine name], [file name]
return null on error
"""
# Sanitize fileArg.
fileArg = ''.join([
x for x in fileArg
if x in string.ascii_letters + string.digits + '_-%'
])
# Translate from String to fileId
if not virtualFile or not FILE_TABLE.has_key(virtualFile):
return None
# For each file that we can export we have to have an entry in the
# global FILE_TABLE
fe = FILE_TABLE[virtualFile]
pathIn = fe.getPathIn(self.cfg.globalParams, clientName, fileArg,
grepString)
pathOut = fe.getPathOut(self.cfg.globalParams, clientName, fileArg,
grepString)
auxGrepString = fe.aux_grep
auxCutString = fe.aux_cut
machine = E.getCrtHostName()
tmpPath = None
# Copy web log from GFS to the log directory if necessary.
if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal(
fileArg, clientName)
if not ok or not os.path.exists(pathIn):
logging.error('Failed on CopyRawReportFromGfsToLocal()')
return None
# Copy the feed log from GFS to the log directory if necessary.
elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
tmpPath = pathOut + "_fromGFS"
(status, output) = E.run_fileutil_command(
self.cfg.globalParams, "cat %s > %s" % (pathIn, tmpPath), 5)
if E.ERR_OK != status:
logging.error("Failed to copy %s to %s" % (pathIn, tmpPath))
return None
pathIn = tmpPath
# Count the files that we can get
files = E.ls([machine], pathIn)
if not files:
numFiles = 0
else:
numFiles = len(files)
# If we need only one file, and there are more than one, use the
# last one.
if numFiles > 0 and not fe.multi_files:
pathIn = files[-1]
# Create the auxiliary command to create the actual file
command = None
if numFiles == 0:
# No files availavle.
command = "echo -e '' > %s" % pathOut
else:
if virtualFile == 'FEED_LOG':
command = "tail -n +2 %s " % pathIn
if fe.do_tac:
command = command + " | tac "
# If we reverse the files before displaying
elif fe.do_tac:
command = "tac `ls -r %s` " % pathIn
# Grep the lines we want
if auxGrepString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | grep -- %s" % commands.mkarg(
auxGrepString)
# Maybe we need another grep
if grepString:
if not command:
command = "cat `ls -r %s`" % pathIn
parsedGrepString = commands.mkarg(grepString)
command = command + " | grep -i -F -- %s" % parsedGrepString
# Maybe a cut as well
if auxCutString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | cut %s" % auxCutString
if command:
command = command + " > " + pathOut
# execute the command "file+operation > temp_filename"
# if the command is null just use the path we have.
if not command:
return pathIn
E.execute([machine], command, None, false)
if tmpPath:
E.rm([machine], tmpPath)
return pathOut
def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg):
"""
Makes a physical file from a virtual one
Creates a temp file with results from grep operation/cating of files
Returns: [machine name], [file name]
return null on error
"""
# Sanitize fileArg.
fileArg = ''.join([x for x in fileArg
if x in string.ascii_letters + string.digits + '_-%'])
# Translate from String to fileId
if not virtualFile or not FILE_TABLE.has_key(virtualFile):
return None
# For each file that we can export we have to have an entry in the
# global FILE_TABLE
fe = FILE_TABLE[virtualFile]
pathIn = fe.getPathIn(self.cfg.globalParams,
clientName, fileArg, grepString)
pathOut = fe.getPathOut(self.cfg.globalParams,
clientName, fileArg, grepString)
auxGrepString = fe.aux_grep
auxCutString = fe.aux_cut
machine = E.getCrtHostName()
tmpPath = None
# Copy web log from GFS to the log directory if necessary.
if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal(fileArg, clientName)
if not ok or not os.path.exists(pathIn):
logging.error('Failed on CopyRawReportFromGfsToLocal()')
return None
# Copy the feed log from GFS to the log directory if necessary.
elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
tmpPath = pathOut + "_fromGFS"
(status, output) = E.run_fileutil_command(self.cfg.globalParams,
"cat %s > %s" % (pathIn, tmpPath), 5)
if E.ERR_OK != status:
logging.error("Failed to copy %s to %s" % (pathIn, tmpPath))
return None
pathIn = tmpPath
# Count the files that we can get
files = E.ls([machine], pathIn)
if not files:
numFiles = 0
else:
numFiles = len(files)
# If we need only one file, and there are more than one, use the
# last one.
if numFiles > 0 and not fe.multi_files:
pathIn = files[-1]
# Create the auxiliary command to create the actual file
command = None
if numFiles == 0 :
# No files availavle.
command = "echo -e '' > %s" % pathOut;
else:
if virtualFile == 'FEED_LOG':
command = "tail -n +2 %s " % pathIn
if fe.do_tac:
command = command + " | tac "
# If we reverse the files before displaying
elif fe.do_tac:
command = "tac `ls -r %s` " % pathIn
# Grep the lines we want
if auxGrepString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | grep -- %s" % commands.mkarg(auxGrepString)
# Maybe we need another grep
if grepString:
if not command:
command = "cat `ls -r %s`" % pathIn
parsedGrepString = commands.mkarg(grepString)
command = command + " | grep -i -F -- %s" % parsedGrepString
# Maybe a cut as well
if auxCutString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | cut %s" % auxCutString
if command:
command = command + " > " + pathOut;
# execute the command "file+operation > temp_filename"
# if the command is null just use the path we have.
if not command:
return pathIn
E.execute([machine], command, None, false);
if tmpPath:
E.rm([machine], tmpPath)
return pathOut
