def setcert(self, certBody):
""" Takes a cert file body as the input, and saves it
as the staging certificate
returns 0 on success, or 1 on failure
"""
retval = 0
self.updatelock.acquire()
try:
try:
open(ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(certBody)
except IOError:
retval = 1
logging.error(
"Couldn't save certificate to [%s]"
% (ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
)
if retval == 0:
verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
verifycode = E.execute(["localhost"], verifycmd, outputList, 60)
if verifycode != 0:
retval = 1
E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't verify certificate [%s]; error code: %d" % (str(outputList), verifycode))
finally:
self.updatelock.release()
return "%d" % retval
def installkey(self):
""" installs the staging key as the currently installed private key
returns:
0 on success,
1 on empty install key (not an error)
2 when the private key is invalid
3 when the private key could not be distributed
"""
self.updatelock.acquire()
try:
# first verify if the staging key is empty (not an error)
if (
not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
) or 0 == len(open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()):
return "1"
# next verify that the staging key is a valid file
verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
verifycode = E.execute(["localhost"], verifycmd, outputList, 60)
if verifycode != 0:
E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Verify failed for key [%s]; error code: %d" % (str(outputList), verifycode))
return "2"
# distribute the staging key
retcode = E.distribute(
self.cfg.getGlobalParam("MACHINES"),
ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"),
60,
)
if retcode != 0:
logging.error("Couldn't distribute private key, error %d" % retcode)
return "3"
# next, copy the key on all machines
cmd = "secure_script_wrapper -p2 %s installkey %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60)
if retcode != 0:
logging.error("Couldn't install cert: %s" % str(outputList))
return "3"
self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED)
finally:
self.updatelock.release()
return "0"
def installkey(self):
""" installs the staging key as the currently installed private key
returns:
0 on success,
1 on empty install key (not an error)
2 when the private key is invalid
3 when the private key could not be distributed
"""
self.updatelock.acquire()
try:
# first verify if the staging key is empty (not an error)
if (not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % \
self.cfg.getGlobalParam("ENTERPRISE_HOME"))) or \
0 == len(open(ssl_cert.STAGINGKEY_FILENAME % \
self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()):
return "1"
# next verify that the staging key is a valid file
verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
outputList = []
verifycode = E.execute(['localhost'], verifycmd, outputList, 60)
if verifycode != 0:
E.rm(['localhost'],
ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Verify failed for key [%s]; error code: %d" %
(str(outputList), verifycode) )
return "2"
# distribute the staging key
retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"),
ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60)
if retcode != 0:
logging.error("Couldn't distribute private key, error %d" % retcode)
return "3"
# next, copy the key on all machines
cmd = "secure_script_wrapper -p2 %s installkey %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))
outputList = []
retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList,
60)
if retcode != 0:
logging.error("Couldn't install cert: %s" % str(outputList))
return "3"
self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED)
finally:
self.updatelock.release()
return "0"
def installcert(self):
""" installs the staging certificate as the currently installed certificate
returns:
0 on success, and
1 on failure
"""
self.updatelock.acquire()
try:
# first verify that the staging certificate is a valid file
verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
verifycode = E.execute(["localhost"], verifycmd, outputList, 60)
if verifycode != 0:
E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Verify failed for certificate [%s]; error code: %d" % (str(outputList), verifycode))
return "1"
# distribute the staging certificate
retcode = E.distribute(
self.cfg.getGlobalParam("MACHINES"),
ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"),
60,
)
if retcode != 0:
logging.error("Couldn't distribute apache cert, error %d" % retcode)
# next, generate the certificate on all machines
cmd = "secure_script_wrapper -p2 %s installcert %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60)
if retcode != 0:
logging.error("Couldn't install cert: %s" % str(outputList))
return "1"
self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_CERT_INSTALLED)
finally:
self.updatelock.release()
return "0"
def installcert(self):
""" installs the staging certificate as the currently installed certificate
returns:
0 on success, and
1 on failure
"""
self.updatelock.acquire()
try:
# first verify that the staging certificate is a valid file
verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
outputList = []
verifycode = E.execute(['localhost'], verifycmd, outputList, 60)
if verifycode != 0:
E.rm(['localhost'],
ssl_cert.STAGINGCERT_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Verify failed for certificate [%s]; error code: %d" %
(str(outputList), verifycode) )
return "1"
# distribute the staging certificate
retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"),
ssl_cert.STAGINGCERT_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60)
if retcode != 0:
logging.error("Couldn't distribute apache cert, error %d" % retcode)
# next, generate the certificate on all machines
cmd = "secure_script_wrapper -p2 %s installcert %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))
outputList = []
retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList,
60)
if retcode != 0:
logging.error("Couldn't install cert: %s" % str(outputList))
return "1"
self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_CERT_INSTALLED)
finally:
self.updatelock.release()
return "0"
def CollectLogs(all_machines, gws_log_dir, log_collect_dir):
# We only run this on oneway or master node of cluster.
master = find_master.FindMaster(2100, all_machines)
crt_machine = E.getCrtHostName()
if len(all_machines) != 1 and (len(master) != 1 or master[0] != crt_machine):
logging.info('Not a oneway or cluster master node. Return!')
return
lockfile = '%s/lock' % log_collect_dir
# waiting up to 5 minutes for the lock.
lock = E.acquire_lock(lockfile, 30, breakLockAfterGracePeriod = 0)
if lock == None:
logging.info('Cannot grab the lock. Return!')
return
try:
for machine in all_machines:
src_pattern = '%s/partnerlog.*' % gws_log_dir
dest_dir = '%s/%s' % (log_collect_dir, machine)
# If it's a oneway or master node, we make a symlink to gws_log_dir instead
# of rsync to log_collect directory
if machine == crt_machine:
# To make it backward compatible, we need to remove old dest_dir if it's
# already an existing directory from previous version because in previous
# versions we created a dir and rsynced files even on the master node and
# one-ways.
if os.path.exists(dest_dir) and not os.path.islink(dest_dir):
if not E.rm(master, '%s/*' % dest_dir) or not E.rmdir(master, dest_dir):
logging.error('Directory %s exists and cannot be cleaned.', dest_dir)
continue
logging.info('Cleaned existing directory %s.', dest_dir)
if E.ln(master, gws_log_dir, dest_dir):
logging.info('Symlink %s to directory %s:%s for logs' %
(dest_dir, machine, gws_log_dir))
else:
logging.error('Cannot make a symlink from %s to %s' %
(dest_dir, gws_log_dir))
continue
# For non-master nodes on cluster, we need to rsync those files to master node
logging.info('Collecting logs from %s:%s into %s' % (
machine, src_pattern, dest_dir))
# make log directories if needed
liblog.MakeDir(dest_dir)
# rsync all files from one remote machine in one command.
rsync_cmd = 'rsync --timeout=60 --size-only -vau ' \
' -e ssh %s:%s %s/' % (machine, src_pattern, dest_dir)
# rsync the logs
(status, output) = liblog.DoCommand(rsync_cmd)
if status != 0:
logging.error('Failed to collect logs from %s: %s' % (
machine, output))
finally:
lock.close()
os.unlink(lockfile)
def distributedeletedbfile(self, filename):
# Remove local database files from all the nodes.
machines = self.cfg.getGlobalParam("MACHINES")
if not E.rm(machines, filename):
logging.error("Failed to delete %s on %s" % (filename, machines))
return "1"
return "0"
def importkey(self, keyBody):
""" Takes a private key file body as the input, and saves it
as the staging key
returns 0 on success, or 1 on failure
"""
retval = 0
self.updatelock.acquire()
try:
try:
open(ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"), 'w').write(keyBody)
except IOError:
retval = 1
logging.error("Couldn't save key to [%s]" %
(ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME")))
if retval == 0:
# check the key
if keyBody:
verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
outputList = []
verifycode = E.execute(['localhost'], verifycmd, outputList, 60)
else:
verifycode = 0
if verifycode != 0:
retval = 1
E.rm(['localhost'],
ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't verify key [%s]; error code: %d" %
(str(outputList), verifycode) )
finally:
self.updatelock.release()
return "%d" % retval
def importcrl(self, crldata):
'''Import a CRL'''
if not os.path.exists(self.cfg.getGlobalParam('CRL_DIRNAME')):
os.mkdir(self.cfg.getGlobalParam('CRL_DIRNAME'))
tempfilename = os.path.join(
self.cfg.getGlobalParam('CRL_DIRNAME'), 'temp')
retcode = 0
self.updatelock.acquire()
try:
try:
open(tempfilename, 'w').write(crldata)
except IOError:
logging.error('Could not write CRL data to [%s]' % tempfilename)
return '-4'
retcode, result = E.getstatusoutput(
'%s importcrl %s %s %s' %
(self.sslWrapperPath, tempfilename,
self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME'),
self.cfg.getGlobalParam('CRL_DIRNAME')
))
retcode = retcode / 256
if retcode != 0:
return result
retcode = self._distributeFiles(self.cfg.getGlobalParam('CRL_DIRNAME'))
if retcode != 0:
logging.error('Error distributing CRL file: %d' % retcode)
return str(retcode)
finally:
if os.path.exists(tempfilename):
E.rm(['localhost'], tempfilename)
self.updatelock.release()
return '0'
def importkey(self, keyBody):
""" Takes a private key file body as the input, and saves it
as the staging key
returns 0 on success, or 1 on failure
"""
retval = 0
self.updatelock.acquire()
try:
try:
open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(keyBody)
except IOError:
retval = 1
logging.error(
"Couldn't save key to [%s]"
% (ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
)
if retval == 0:
# check the key
if keyBody:
verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
verifycode = E.execute(["localhost"], verifycmd, outputList, 60)
else:
verifycode = 0
if verifycode != 0:
retval = 1
E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't verify key [%s]; error code: %d" % (str(outputList), verifycode))
finally:
self.updatelock.release()
return "%d" % retval
def importcas(self, cadata):
''' This will import the list of trusted CAs in @cadata'''
trusted_ca_dir = self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME')
if not os.path.exists(trusted_ca_dir):
os.mkdir(trusted_ca_dir)
## write cadata to a temp file
tempfilename = os.path.join(trusted_ca_dir, 'temp')
self.updatelock.acquire()
try:
try:
open(tempfilename, 'w').write(cadata)
except IOError:
logging.error('Could not write CA data to [%s]' % tempfilename)
return '-4'
retcode, result = E.getstatusoutput(
'%s importcas %s %s' %
(self.sslWrapperPath, tempfilename, trusted_ca_dir))
retcode = retcode / 256
if retcode != 0:
return result
# distribute CA files to all nodes in the network.
retcode = self._distributeFiles(trusted_ca_dir)
if retcode != 0:
logging.error('Error distributing CA file: %d' % retcode)
return str(retcode)
finally:
## check for existence?
E.rm(['localhost'], tempfilename)
self.updatelock.release()
return '0'
def generatekey(self):
""" creates a randomly generated staging key; returns 0 on success, 1 on
failure """
self.updatelock.acquire()
try:
cmd = "secure_script_wrapper -p2 %s generatekey %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))
outputList = []
retcode = E.execute(['localhost'], cmd, outputList, 60)
# if the command failed, we don't want to leave a malformed key around
if retcode != 0:
E.rm(['localhost'], ssl_cert.STAGINGKEY_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't generate private key: %s" %str(outputList))
finally:
self.updatelock.release()
return "%d" % retcode
def setcert(self, certBody):
""" Takes a cert file body as the input, and saves it
as the staging certificate
returns 0 on success, or 1 on failure
"""
retval = 0
self.updatelock.acquire()
try:
try:
open(ssl_cert.STAGINGCERT_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"), 'w').write(certBody)
except IOError:
retval = 1
logging.error("Couldn't save certificate to [%s]" %
(ssl_cert.STAGINGCERT_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME")))
if retval == 0:
verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
outputList = []
verifycode = E.execute(['localhost'], verifycmd, outputList, 60)
if verifycode != 0:
retval = 1
E.rm(['localhost'],
ssl_cert.STAGINGCERT_FILENAME %
self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't verify certificate [%s]; error code: %d" %
(str(outputList), verifycode) )
finally:
self.updatelock.release()
return "%d" % retval
def generatekey(self):
""" creates a randomly generated staging key; returns 0 on success, 1 on
failure """
self.updatelock.acquire()
try:
cmd = "secure_script_wrapper -p2 %s generatekey %s" % (
self.sslWrapperPath,
self.cfg.getGlobalParam("ENTERPRISE_HOME"),
)
outputList = []
retcode = E.execute(["localhost"], cmd, outputList, 60)
# if the command failed, we don't want to leave a malformed key around
if retcode != 0:
E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
logging.error("Couldn't generate private key: %s" % str(outputList))
finally:
self.updatelock.release()
return "%d" % retcode
def removeca(self, hash):
'''Remove a trusted CA, given its subject hash'''
self.updatelock.acquire()
retval = 0
try:
name = os.path.join(self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME'),
hash + ssl_cert.CA_CERT_EXT)
retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name)
if not retval:
logging.error('error trying to remove %s: %d' % (name, retval))
name = os.path.join(self.cfg.getGlobalParam('CRL_DIRNAME'),
hash + ssl_cert.CRL_EXT)
retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name)
if not retval:
logging.info('error trying to remove %s: %d' % (name, retval))
finally:
self.updatelock.release()
return '%d' % retval
def create(linkFarmDir, chunkDisksMap, chunkPrefix, chunkTermination, binDir, machines):
"""
This does the create job.
@param linkFarmDir the directory where is the link frm. Here we create
a data link, a log dir and log dir
@param chunkDisks the map from machines to datadisks
@param chunkPrefix
@param chunkTermination the dir termination
@param machines on which machines we create this datadir
@return boolean - the succes status
"""
ok = 1
for machine in machines:
if not chunkDisksMap.has_key(machine):
logging.error("ERROR: No entry for machine %s in data chunk disk map %s" % (machine, chunkDisksMap))
ok = 0
continue
chunkDisks = chunkDisksMap[machine]
# Prepare search.config content
searchConfig = []
searchConfig.append(
"""datapath %s
urlbuckets urlbuckets
sorttempdir .
"""
% chunkTermination
)
dirs = []
for d in ["-data", "-data/logs", "-data/%s" % C.RUNCONFIGDIR_DIRNAME]:
dirs.append("%s/%s%s" % (linkFarmDir, chunkTermination, d))
for d in chunkDisks:
searchConfig.append("disk %s/%s 1000\n" % (d, chunkPrefix))
dirs.append("%s/%s/%s" % (d, chunkPrefix, chunkTermination))
dirs.append("%s/%s/workqueue" % (d, chunkPrefix))
dirs.append("%s/%s/workqueue/bin" % (d, chunkPrefix))
if not E.mkdir([machine], string.join(dirs, " ")) or not E.ln(
[machine],
"%s/%s-data" % (linkFarmDir, chunkTermination),
"%s/%s-data/data" % (linkFarmDir, chunkTermination),
):
return false
# create the search.config and distribute it
fileName = "%s/%s-data/search.config" % (linkFarmDir, chunkTermination)
tmpFile = "/tmp/search.config.tmp-%s" % chunkTermination
try:
open(tmpFile, "w").write(string.join(searchConfig, ""))
if E.ERR_OK != E.distribute([machine], tmpFile, 1) or E.ERR_OK != E.execute(
[machine], "mv -f %s %s" % (tmpFile, fileName), None, true
):
ok = 0
E.rm([E.LOCALHOST, machine], tmpFile)
except IOError:
ok = 0
# set up link in workqueue slave binary directory (bin) to workqueue-slave
# binary so that workqueue-slave can checksum/update itself
for d in chunkDisks:
if not E.ln(
[machine],
"%s/workqueue-slave" % binDir["workqueue-slave"],
"%s/%s/workqueue/bin/current" % (d, chunkPrefix),
):
ok = 0
return ok
def create(linkFarmDir,
chunkDisksMap, chunkPrefix, chunkTermination,
binDir,
machines):
"""
This does the create job.
@param linkFarmDir the directory where is the link frm. Here we create
a data link, a log dir and log dir
@param chunkDisks the map from machines to datadisks
@param chunkPrefix
@param chunkTermination the dir termination
@param machines on which machines we create this datadir
@return boolean - the succes status
"""
ok = 1
for machine in machines:
if not chunkDisksMap.has_key(machine):
logging.error(
"ERROR: No entry for machine %s in data chunk disk map %s" % (
machine, chunkDisksMap))
ok = 0
continue
chunkDisks = chunkDisksMap[machine]
# Prepare search.config content
searchConfig = []
searchConfig.append("""datapath %s
urlbuckets urlbuckets
sorttempdir .
""" % chunkTermination)
dirs = []
for d in ["-data", "-data/logs", "-data/%s" % C.RUNCONFIGDIR_DIRNAME]:
dirs.append("%s/%s%s" % (linkFarmDir, chunkTermination, d))
for d in chunkDisks:
searchConfig.append("disk %s/%s 1000\n" % (d, chunkPrefix))
dirs.append("%s/%s/%s" % (d, chunkPrefix, chunkTermination))
dirs.append("%s/%s/workqueue" % (d, chunkPrefix))
dirs.append("%s/%s/workqueue/bin" % (d, chunkPrefix))
if ( not E.mkdir([machine], string.join(dirs, " ")) or
not E.ln([machine],
"%s/%s-data" % (linkFarmDir, chunkTermination),
"%s/%s-data/data" % (linkFarmDir, chunkTermination)) ):
return false
# create the search.config and distribute it
fileName = "%s/%s-data/search.config" % (linkFarmDir,
chunkTermination)
tmpFile = "/tmp/search.config.tmp-%s" % chunkTermination;
try:
open(tmpFile, "w").write(string.join(searchConfig, ""))
if ( E.ERR_OK != E.distribute([machine], tmpFile, 1) or
E.ERR_OK != E.execute([machine],
"mv -f %s %s" % (tmpFile, fileName),
None, true) ):
ok = 0
E.rm([E.LOCALHOST, machine], tmpFile)
except IOError:
ok = 0
# set up link in workqueue slave binary directory (bin) to workqueue-slave
# binary so that workqueue-slave can checksum/update itself
for d in chunkDisks:
if not E.ln([machine],
'%s/workqueue-slave' % binDir['workqueue-slave'],
'%s/%s/workqueue/bin/current' % (d, chunkPrefix)):
ok = 0
return ok;
def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg):
"""
Makes a physical file from a virtual one
Creates a temp file with results from grep operation/cating of files
Returns: [machine name], [file name]
return null on error
"""
# Sanitize fileArg.
fileArg = ''.join([
x for x in fileArg
if x in string.ascii_letters + string.digits + '_-%'
])
# Translate from String to fileId
if not virtualFile or not FILE_TABLE.has_key(virtualFile):
return None
# For each file that we can export we have to have an entry in the
# global FILE_TABLE
fe = FILE_TABLE[virtualFile]
pathIn = fe.getPathIn(self.cfg.globalParams, clientName, fileArg,
grepString)
pathOut = fe.getPathOut(self.cfg.globalParams, clientName, fileArg,
grepString)
auxGrepString = fe.aux_grep
auxCutString = fe.aux_cut
machine = E.getCrtHostName()
tmpPath = None
# Copy web log from GFS to the log directory if necessary.
if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal(
fileArg, clientName)
if not ok or not os.path.exists(pathIn):
logging.error('Failed on CopyRawReportFromGfsToLocal()')
return None
# Copy the feed log from GFS to the log directory if necessary.
elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
tmpPath = pathOut + "_fromGFS"
(status, output) = E.run_fileutil_command(
self.cfg.globalParams, "cat %s > %s" % (pathIn, tmpPath), 5)
if E.ERR_OK != status:
logging.error("Failed to copy %s to %s" % (pathIn, tmpPath))
return None
pathIn = tmpPath
# Count the files that we can get
files = E.ls([machine], pathIn)
if not files:
numFiles = 0
else:
numFiles = len(files)
# If we need only one file, and there are more than one, use the
# last one.
if numFiles > 0 and not fe.multi_files:
pathIn = files[-1]
# Create the auxiliary command to create the actual file
command = None
if numFiles == 0:
# No files availavle.
command = "echo -e '' > %s" % pathOut
else:
if virtualFile == 'FEED_LOG':
command = "tail -n +2 %s " % pathIn
if fe.do_tac:
command = command + " | tac "
# If we reverse the files before displaying
elif fe.do_tac:
command = "tac `ls -r %s` " % pathIn
# Grep the lines we want
if auxGrepString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | grep -- %s" % commands.mkarg(
auxGrepString)
# Maybe we need another grep
if grepString:
if not command:
command = "cat `ls -r %s`" % pathIn
parsedGrepString = commands.mkarg(grepString)
command = command + " | grep -i -F -- %s" % parsedGrepString
# Maybe a cut as well
if auxCutString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | cut %s" % auxCutString
if command:
command = command + " > " + pathOut
# execute the command "file+operation > temp_filename"
# if the command is null just use the path we have.
if not command:
return pathIn
E.execute([machine], command, None, false)
if tmpPath:
E.rm([machine], tmpPath)
return pathOut
def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg):
"""
Makes a physical file from a virtual one
Creates a temp file with results from grep operation/cating of files
Returns: [machine name], [file name]
return null on error
"""
# Sanitize fileArg.
fileArg = ''.join([x for x in fileArg
if x in string.ascii_letters + string.digits + '_-%'])
# Translate from String to fileId
if not virtualFile or not FILE_TABLE.has_key(virtualFile):
return None
# For each file that we can export we have to have an entry in the
# global FILE_TABLE
fe = FILE_TABLE[virtualFile]
pathIn = fe.getPathIn(self.cfg.globalParams,
clientName, fileArg, grepString)
pathOut = fe.getPathOut(self.cfg.globalParams,
clientName, fileArg, grepString)
auxGrepString = fe.aux_grep
auxCutString = fe.aux_cut
machine = E.getCrtHostName()
tmpPath = None
# Copy web log from GFS to the log directory if necessary.
if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal(fileArg, clientName)
if not ok or not os.path.exists(pathIn):
logging.error('Failed on CopyRawReportFromGfsToLocal()')
return None
# Copy the feed log from GFS to the log directory if necessary.
elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \
not os.path.exists(pathIn):
tmpPath = pathOut + "_fromGFS"
(status, output) = E.run_fileutil_command(self.cfg.globalParams,
"cat %s > %s" % (pathIn, tmpPath), 5)
if E.ERR_OK != status:
logging.error("Failed to copy %s to %s" % (pathIn, tmpPath))
return None
pathIn = tmpPath
# Count the files that we can get
files = E.ls([machine], pathIn)
if not files:
numFiles = 0
else:
numFiles = len(files)
# If we need only one file, and there are more than one, use the
# last one.
if numFiles > 0 and not fe.multi_files:
pathIn = files[-1]
# Create the auxiliary command to create the actual file
command = None
if numFiles == 0 :
# No files availavle.
command = "echo -e '' > %s" % pathOut;
else:
if virtualFile == 'FEED_LOG':
command = "tail -n +2 %s " % pathIn
if fe.do_tac:
command = command + " | tac "
# If we reverse the files before displaying
elif fe.do_tac:
command = "tac `ls -r %s` " % pathIn
# Grep the lines we want
if auxGrepString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | grep -- %s" % commands.mkarg(auxGrepString)
# Maybe we need another grep
if grepString:
if not command:
command = "cat `ls -r %s`" % pathIn
parsedGrepString = commands.mkarg(grepString)
command = command + " | grep -i -F -- %s" % parsedGrepString
# Maybe a cut as well
if auxCutString:
if not command:
command = "cat `ls -r %s`" % pathIn
command = command + " | cut %s" % auxCutString
if command:
command = command + " > " + pathOut;
# execute the command "file+operation > temp_filename"
# if the command is null just use the path we have.
if not command:
return pathIn
E.execute([machine], command, None, false);
if tmpPath:
E.rm([machine], tmpPath)
return pathOut
