def setcert(self, certBody): """ Takes a cert file body as the input, and saves it as the staging certificate returns 0 on success, or 1 on failure """ retval = 0 self.updatelock.acquire() try: try: open(ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(certBody) except IOError: retval = 1 logging.error( "Couldn't save certificate to [%s]" % (ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) ) if retval == 0: verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] verifycode = E.execute(["localhost"], verifycmd, outputList, 60) if verifycode != 0: retval = 1 E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't verify certificate [%s]; error code: %d" % (str(outputList), verifycode)) finally: self.updatelock.release() return "%d" % retval
def installkey(self): """ installs the staging key as the currently installed private key returns: 0 on success, 1 on empty install key (not an error) 2 when the private key is invalid 3 when the private key could not be distributed """ self.updatelock.acquire() try: # first verify if the staging key is empty (not an error) if ( not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) ) or 0 == len(open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()): return "1" # next verify that the staging key is a valid file verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] verifycode = E.execute(["localhost"], verifycmd, outputList, 60) if verifycode != 0: E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Verify failed for key [%s]; error code: %d" % (str(outputList), verifycode)) return "2" # distribute the staging key retcode = E.distribute( self.cfg.getGlobalParam("MACHINES"), ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60, ) if retcode != 0: logging.error("Couldn't distribute private key, error %d" % retcode) return "3" # next, copy the key on all machines cmd = "secure_script_wrapper -p2 %s installkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60) if retcode != 0: logging.error("Couldn't install cert: %s" % str(outputList)) return "3" self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED) finally: self.updatelock.release() return "0"
def installkey(self): """ installs the staging key as the currently installed private key returns: 0 on success, 1 on empty install key (not an error) 2 when the private key is invalid 3 when the private key could not be distributed """ self.updatelock.acquire() try: # first verify if the staging key is empty (not an error) if (not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % \ self.cfg.getGlobalParam("ENTERPRISE_HOME"))) or \ 0 == len(open(ssl_cert.STAGINGKEY_FILENAME % \ self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()): return "1" # next verify that the staging key is a valid file verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") ) outputList = [] verifycode = E.execute(['localhost'], verifycmd, outputList, 60) if verifycode != 0: E.rm(['localhost'], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Verify failed for key [%s]; error code: %d" % (str(outputList), verifycode) ) return "2" # distribute the staging key retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"), ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60) if retcode != 0: logging.error("Couldn't distribute private key, error %d" % retcode) return "3" # next, copy the key on all machines cmd = "secure_script_wrapper -p2 %s installkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME")) outputList = [] retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60) if retcode != 0: logging.error("Couldn't install cert: %s" % str(outputList)) return "3" self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED) finally: self.updatelock.release() return "0"
def installcert(self): """ installs the staging certificate as the currently installed certificate returns: 0 on success, and 1 on failure """ self.updatelock.acquire() try: # first verify that the staging certificate is a valid file verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] verifycode = E.execute(["localhost"], verifycmd, outputList, 60) if verifycode != 0: E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Verify failed for certificate [%s]; error code: %d" % (str(outputList), verifycode)) return "1" # distribute the staging certificate retcode = E.distribute( self.cfg.getGlobalParam("MACHINES"), ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60, ) if retcode != 0: logging.error("Couldn't distribute apache cert, error %d" % retcode) # next, generate the certificate on all machines cmd = "secure_script_wrapper -p2 %s installcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60) if retcode != 0: logging.error("Couldn't install cert: %s" % str(outputList)) return "1" self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_CERT_INSTALLED) finally: self.updatelock.release() return "0"
def installcert(self): """ installs the staging certificate as the currently installed certificate returns: 0 on success, and 1 on failure """ self.updatelock.acquire() try: # first verify that the staging certificate is a valid file verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") ) outputList = [] verifycode = E.execute(['localhost'], verifycmd, outputList, 60) if verifycode != 0: E.rm(['localhost'], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Verify failed for certificate [%s]; error code: %d" % (str(outputList), verifycode) ) return "1" # distribute the staging certificate retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"), ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60) if retcode != 0: logging.error("Couldn't distribute apache cert, error %d" % retcode) # next, generate the certificate on all machines cmd = "secure_script_wrapper -p2 %s installcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME")) outputList = [] retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList, 60) if retcode != 0: logging.error("Couldn't install cert: %s" % str(outputList)) return "1" self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_CERT_INSTALLED) finally: self.updatelock.release() return "0"
def CollectLogs(all_machines, gws_log_dir, log_collect_dir): # We only run this on oneway or master node of cluster. master = find_master.FindMaster(2100, all_machines) crt_machine = E.getCrtHostName() if len(all_machines) != 1 and (len(master) != 1 or master[0] != crt_machine): logging.info('Not a oneway or cluster master node. Return!') return lockfile = '%s/lock' % log_collect_dir # waiting up to 5 minutes for the lock. lock = E.acquire_lock(lockfile, 30, breakLockAfterGracePeriod = 0) if lock == None: logging.info('Cannot grab the lock. Return!') return try: for machine in all_machines: src_pattern = '%s/partnerlog.*' % gws_log_dir dest_dir = '%s/%s' % (log_collect_dir, machine) # If it's a oneway or master node, we make a symlink to gws_log_dir instead # of rsync to log_collect directory if machine == crt_machine: # To make it backward compatible, we need to remove old dest_dir if it's # already an existing directory from previous version because in previous # versions we created a dir and rsynced files even on the master node and # one-ways. if os.path.exists(dest_dir) and not os.path.islink(dest_dir): if not E.rm(master, '%s/*' % dest_dir) or not E.rmdir(master, dest_dir): logging.error('Directory %s exists and cannot be cleaned.', dest_dir) continue logging.info('Cleaned existing directory %s.', dest_dir) if E.ln(master, gws_log_dir, dest_dir): logging.info('Symlink %s to directory %s:%s for logs' % (dest_dir, machine, gws_log_dir)) else: logging.error('Cannot make a symlink from %s to %s' % (dest_dir, gws_log_dir)) continue # For non-master nodes on cluster, we need to rsync those files to master node logging.info('Collecting logs from %s:%s into %s' % ( machine, src_pattern, dest_dir)) # make log directories if needed liblog.MakeDir(dest_dir) # rsync all files from one remote machine in one command. rsync_cmd = 'rsync --timeout=60 --size-only -vau ' \ ' -e ssh %s:%s %s/' % (machine, src_pattern, dest_dir) # rsync the logs (status, output) = liblog.DoCommand(rsync_cmd) if status != 0: logging.error('Failed to collect logs from %s: %s' % ( machine, output)) finally: lock.close() os.unlink(lockfile)
def distributedeletedbfile(self, filename): # Remove local database files from all the nodes. machines = self.cfg.getGlobalParam("MACHINES") if not E.rm(machines, filename): logging.error("Failed to delete %s on %s" % (filename, machines)) return "1" return "0"
def importkey(self, keyBody): """ Takes a private key file body as the input, and saves it as the staging key returns 0 on success, or 1 on failure """ retval = 0 self.updatelock.acquire() try: try: open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 'w').write(keyBody) except IOError: retval = 1 logging.error("Couldn't save key to [%s]" % (ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))) if retval == 0: # check the key if keyBody: verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") ) outputList = [] verifycode = E.execute(['localhost'], verifycmd, outputList, 60) else: verifycode = 0 if verifycode != 0: retval = 1 E.rm(['localhost'], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't verify key [%s]; error code: %d" % (str(outputList), verifycode) ) finally: self.updatelock.release() return "%d" % retval
def importcrl(self, crldata): '''Import a CRL''' if not os.path.exists(self.cfg.getGlobalParam('CRL_DIRNAME')): os.mkdir(self.cfg.getGlobalParam('CRL_DIRNAME')) tempfilename = os.path.join( self.cfg.getGlobalParam('CRL_DIRNAME'), 'temp') retcode = 0 self.updatelock.acquire() try: try: open(tempfilename, 'w').write(crldata) except IOError: logging.error('Could not write CRL data to [%s]' % tempfilename) return '-4' retcode, result = E.getstatusoutput( '%s importcrl %s %s %s' % (self.sslWrapperPath, tempfilename, self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME'), self.cfg.getGlobalParam('CRL_DIRNAME') )) retcode = retcode / 256 if retcode != 0: return result retcode = self._distributeFiles(self.cfg.getGlobalParam('CRL_DIRNAME')) if retcode != 0: logging.error('Error distributing CRL file: %d' % retcode) return str(retcode) finally: if os.path.exists(tempfilename): E.rm(['localhost'], tempfilename) self.updatelock.release() return '0'
def importkey(self, keyBody): """ Takes a private key file body as the input, and saves it as the staging key returns 0 on success, or 1 on failure """ retval = 0 self.updatelock.acquire() try: try: open(ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(keyBody) except IOError: retval = 1 logging.error( "Couldn't save key to [%s]" % (ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) ) if retval == 0: # check the key if keyBody: verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] verifycode = E.execute(["localhost"], verifycmd, outputList, 60) else: verifycode = 0 if verifycode != 0: retval = 1 E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't verify key [%s]; error code: %d" % (str(outputList), verifycode)) finally: self.updatelock.release() return "%d" % retval
def importcas(self, cadata): ''' This will import the list of trusted CAs in @cadata''' trusted_ca_dir = self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME') if not os.path.exists(trusted_ca_dir): os.mkdir(trusted_ca_dir) ## write cadata to a temp file tempfilename = os.path.join(trusted_ca_dir, 'temp') self.updatelock.acquire() try: try: open(tempfilename, 'w').write(cadata) except IOError: logging.error('Could not write CA data to [%s]' % tempfilename) return '-4' retcode, result = E.getstatusoutput( '%s importcas %s %s' % (self.sslWrapperPath, tempfilename, trusted_ca_dir)) retcode = retcode / 256 if retcode != 0: return result # distribute CA files to all nodes in the network. retcode = self._distributeFiles(trusted_ca_dir) if retcode != 0: logging.error('Error distributing CA file: %d' % retcode) return str(retcode) finally: ## check for existence? E.rm(['localhost'], tempfilename) self.updatelock.release() return '0'
def generatekey(self): """ creates a randomly generated staging key; returns 0 on success, 1 on failure """ self.updatelock.acquire() try: cmd = "secure_script_wrapper -p2 %s generatekey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME")) outputList = [] retcode = E.execute(['localhost'], cmd, outputList, 60) # if the command failed, we don't want to leave a malformed key around if retcode != 0: E.rm(['localhost'], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't generate private key: %s" %str(outputList)) finally: self.updatelock.release() return "%d" % retcode
def setcert(self, certBody): """ Takes a cert file body as the input, and saves it as the staging certificate returns 0 on success, or 1 on failure """ retval = 0 self.updatelock.acquire() try: try: open(ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), 'w').write(certBody) except IOError: retval = 1 logging.error("Couldn't save certificate to [%s]" % (ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))) if retval == 0: verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") ) outputList = [] verifycode = E.execute(['localhost'], verifycmd, outputList, 60) if verifycode != 0: retval = 1 E.rm(['localhost'], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't verify certificate [%s]; error code: %d" % (str(outputList), verifycode) ) finally: self.updatelock.release() return "%d" % retval
def generatekey(self): """ creates a randomly generated staging key; returns 0 on success, 1 on failure """ self.updatelock.acquire() try: cmd = "secure_script_wrapper -p2 %s generatekey %s" % ( self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"), ) outputList = [] retcode = E.execute(["localhost"], cmd, outputList, 60) # if the command failed, we don't want to leave a malformed key around if retcode != 0: E.rm(["localhost"], ssl_cert.STAGINGKEY_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME")) logging.error("Couldn't generate private key: %s" % str(outputList)) finally: self.updatelock.release() return "%d" % retcode
def removeca(self, hash): '''Remove a trusted CA, given its subject hash''' self.updatelock.acquire() retval = 0 try: name = os.path.join(self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME'), hash + ssl_cert.CA_CERT_EXT) retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name) if not retval: logging.error('error trying to remove %s: %d' % (name, retval)) name = os.path.join(self.cfg.getGlobalParam('CRL_DIRNAME'), hash + ssl_cert.CRL_EXT) retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name) if not retval: logging.info('error trying to remove %s: %d' % (name, retval)) finally: self.updatelock.release() return '%d' % retval
def create(linkFarmDir, chunkDisksMap, chunkPrefix, chunkTermination, binDir, machines): """ This does the create job. @param linkFarmDir the directory where is the link frm. Here we create a data link, a log dir and log dir @param chunkDisks the map from machines to datadisks @param chunkPrefix @param chunkTermination the dir termination @param machines on which machines we create this datadir @return boolean - the succes status """ ok = 1 for machine in machines: if not chunkDisksMap.has_key(machine): logging.error("ERROR: No entry for machine %s in data chunk disk map %s" % (machine, chunkDisksMap)) ok = 0 continue chunkDisks = chunkDisksMap[machine] # Prepare search.config content searchConfig = [] searchConfig.append( """datapath %s urlbuckets urlbuckets sorttempdir . """ % chunkTermination ) dirs = [] for d in ["-data", "-data/logs", "-data/%s" % C.RUNCONFIGDIR_DIRNAME]: dirs.append("%s/%s%s" % (linkFarmDir, chunkTermination, d)) for d in chunkDisks: searchConfig.append("disk %s/%s 1000\n" % (d, chunkPrefix)) dirs.append("%s/%s/%s" % (d, chunkPrefix, chunkTermination)) dirs.append("%s/%s/workqueue" % (d, chunkPrefix)) dirs.append("%s/%s/workqueue/bin" % (d, chunkPrefix)) if not E.mkdir([machine], string.join(dirs, " ")) or not E.ln( [machine], "%s/%s-data" % (linkFarmDir, chunkTermination), "%s/%s-data/data" % (linkFarmDir, chunkTermination), ): return false # create the search.config and distribute it fileName = "%s/%s-data/search.config" % (linkFarmDir, chunkTermination) tmpFile = "/tmp/search.config.tmp-%s" % chunkTermination try: open(tmpFile, "w").write(string.join(searchConfig, "")) if E.ERR_OK != E.distribute([machine], tmpFile, 1) or E.ERR_OK != E.execute( [machine], "mv -f %s %s" % (tmpFile, fileName), None, true ): ok = 0 E.rm([E.LOCALHOST, machine], tmpFile) except IOError: ok = 0 # set up link in workqueue slave binary directory (bin) to workqueue-slave # binary so that workqueue-slave can checksum/update itself for d in chunkDisks: if not E.ln( [machine], "%s/workqueue-slave" % binDir["workqueue-slave"], "%s/%s/workqueue/bin/current" % (d, chunkPrefix), ): ok = 0 return ok
def create(linkFarmDir, chunkDisksMap, chunkPrefix, chunkTermination, binDir, machines): """ This does the create job. @param linkFarmDir the directory where is the link frm. Here we create a data link, a log dir and log dir @param chunkDisks the map from machines to datadisks @param chunkPrefix @param chunkTermination the dir termination @param machines on which machines we create this datadir @return boolean - the succes status """ ok = 1 for machine in machines: if not chunkDisksMap.has_key(machine): logging.error( "ERROR: No entry for machine %s in data chunk disk map %s" % ( machine, chunkDisksMap)) ok = 0 continue chunkDisks = chunkDisksMap[machine] # Prepare search.config content searchConfig = [] searchConfig.append("""datapath %s urlbuckets urlbuckets sorttempdir . """ % chunkTermination) dirs = [] for d in ["-data", "-data/logs", "-data/%s" % C.RUNCONFIGDIR_DIRNAME]: dirs.append("%s/%s%s" % (linkFarmDir, chunkTermination, d)) for d in chunkDisks: searchConfig.append("disk %s/%s 1000\n" % (d, chunkPrefix)) dirs.append("%s/%s/%s" % (d, chunkPrefix, chunkTermination)) dirs.append("%s/%s/workqueue" % (d, chunkPrefix)) dirs.append("%s/%s/workqueue/bin" % (d, chunkPrefix)) if ( not E.mkdir([machine], string.join(dirs, " ")) or not E.ln([machine], "%s/%s-data" % (linkFarmDir, chunkTermination), "%s/%s-data/data" % (linkFarmDir, chunkTermination)) ): return false # create the search.config and distribute it fileName = "%s/%s-data/search.config" % (linkFarmDir, chunkTermination) tmpFile = "/tmp/search.config.tmp-%s" % chunkTermination; try: open(tmpFile, "w").write(string.join(searchConfig, "")) if ( E.ERR_OK != E.distribute([machine], tmpFile, 1) or E.ERR_OK != E.execute([machine], "mv -f %s %s" % (tmpFile, fileName), None, true) ): ok = 0 E.rm([E.LOCALHOST, machine], tmpFile) except IOError: ok = 0 # set up link in workqueue slave binary directory (bin) to workqueue-slave # binary so that workqueue-slave can checksum/update itself for d in chunkDisks: if not E.ln([machine], '%s/workqueue-slave' % binDir['workqueue-slave'], '%s/%s/workqueue/bin/current' % (d, chunkPrefix)): ok = 0 return ok;
def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg): """ Makes a physical file from a virtual one Creates a temp file with results from grep operation/cating of files Returns: [machine name], [file name] return null on error """ # Sanitize fileArg. fileArg = ''.join([ x for x in fileArg if x in string.ascii_letters + string.digits + '_-%' ]) # Translate from String to fileId if not virtualFile or not FILE_TABLE.has_key(virtualFile): return None # For each file that we can export we have to have an entry in the # global FILE_TABLE fe = FILE_TABLE[virtualFile] pathIn = fe.getPathIn(self.cfg.globalParams, clientName, fileArg, grepString) pathOut = fe.getPathOut(self.cfg.globalParams, clientName, fileArg, grepString) auxGrepString = fe.aux_grep auxCutString = fe.aux_cut machine = E.getCrtHostName() tmpPath = None # Copy web log from GFS to the log directory if necessary. if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \ not os.path.exists(pathIn): ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal( fileArg, clientName) if not ok or not os.path.exists(pathIn): logging.error('Failed on CopyRawReportFromGfsToLocal()') return None # Copy the feed log from GFS to the log directory if necessary. elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \ not os.path.exists(pathIn): tmpPath = pathOut + "_fromGFS" (status, output) = E.run_fileutil_command( self.cfg.globalParams, "cat %s > %s" % (pathIn, tmpPath), 5) if E.ERR_OK != status: logging.error("Failed to copy %s to %s" % (pathIn, tmpPath)) return None pathIn = tmpPath # Count the files that we can get files = E.ls([machine], pathIn) if not files: numFiles = 0 else: numFiles = len(files) # If we need only one file, and there are more than one, use the # last one. if numFiles > 0 and not fe.multi_files: pathIn = files[-1] # Create the auxiliary command to create the actual file command = None if numFiles == 0: # No files availavle. command = "echo -e '' > %s" % pathOut else: if virtualFile == 'FEED_LOG': command = "tail -n +2 %s " % pathIn if fe.do_tac: command = command + " | tac " # If we reverse the files before displaying elif fe.do_tac: command = "tac `ls -r %s` " % pathIn # Grep the lines we want if auxGrepString: if not command: command = "cat `ls -r %s`" % pathIn command = command + " | grep -- %s" % commands.mkarg( auxGrepString) # Maybe we need another grep if grepString: if not command: command = "cat `ls -r %s`" % pathIn parsedGrepString = commands.mkarg(grepString) command = command + " | grep -i -F -- %s" % parsedGrepString # Maybe a cut as well if auxCutString: if not command: command = "cat `ls -r %s`" % pathIn command = command + " | cut %s" % auxCutString if command: command = command + " > " + pathOut # execute the command "file+operation > temp_filename" # if the command is null just use the path we have. if not command: return pathIn E.execute([machine], command, None, false) if tmpPath: E.rm([machine], tmpPath) return pathOut
def makePhysicalFile(self, virtualFile, clientName, grepString, fileArg): """ Makes a physical file from a virtual one Creates a temp file with results from grep operation/cating of files Returns: [machine name], [file name] return null on error """ # Sanitize fileArg. fileArg = ''.join([x for x in fileArg if x in string.ascii_letters + string.digits + '_-%']) # Translate from String to fileId if not virtualFile or not FILE_TABLE.has_key(virtualFile): return None # For each file that we can export we have to have an entry in the # global FILE_TABLE fe = FILE_TABLE[virtualFile] pathIn = fe.getPathIn(self.cfg.globalParams, clientName, fileArg, grepString) pathOut = fe.getPathOut(self.cfg.globalParams, clientName, fileArg, grepString) auxGrepString = fe.aux_grep auxCutString = fe.aux_cut machine = E.getCrtHostName() tmpPath = None # Copy web log from GFS to the log directory if necessary. if virtualFile == 'WEB_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \ not os.path.exists(pathIn): ok = self.cfg.logmanager.CopyRawReportFromGfsToLocal(fileArg, clientName) if not ok or not os.path.exists(pathIn): logging.error('Failed on CopyRawReportFromGfsToLocal()') return None # Copy the feed log from GFS to the log directory if necessary. elif virtualFile == 'FEED_LOG' and self.cfg.getGlobalParam('GFS_ALIASES') and \ not os.path.exists(pathIn): tmpPath = pathOut + "_fromGFS" (status, output) = E.run_fileutil_command(self.cfg.globalParams, "cat %s > %s" % (pathIn, tmpPath), 5) if E.ERR_OK != status: logging.error("Failed to copy %s to %s" % (pathIn, tmpPath)) return None pathIn = tmpPath # Count the files that we can get files = E.ls([machine], pathIn) if not files: numFiles = 0 else: numFiles = len(files) # If we need only one file, and there are more than one, use the # last one. if numFiles > 0 and not fe.multi_files: pathIn = files[-1] # Create the auxiliary command to create the actual file command = None if numFiles == 0 : # No files availavle. command = "echo -e '' > %s" % pathOut; else: if virtualFile == 'FEED_LOG': command = "tail -n +2 %s " % pathIn if fe.do_tac: command = command + " | tac " # If we reverse the files before displaying elif fe.do_tac: command = "tac `ls -r %s` " % pathIn # Grep the lines we want if auxGrepString: if not command: command = "cat `ls -r %s`" % pathIn command = command + " | grep -- %s" % commands.mkarg(auxGrepString) # Maybe we need another grep if grepString: if not command: command = "cat `ls -r %s`" % pathIn parsedGrepString = commands.mkarg(grepString) command = command + " | grep -i -F -- %s" % parsedGrepString # Maybe a cut as well if auxCutString: if not command: command = "cat `ls -r %s`" % pathIn command = command + " | cut %s" % auxCutString if command: command = command + " > " + pathOut; # execute the command "file+operation > temp_filename" # if the command is null just use the path we have. if not command: return pathIn E.execute([machine], command, None, false); if tmpPath: E.rm([machine], tmpPath) return pathOut