def backupFile(cls, fileP, suffix = ".bak", copy = True):
'''
Creates a backup of the chosen file in the same directory as the file itself.
If the file does not exist or isn't accessible then a BackupException will be raised.
If a backup file already exists for the file and it wasn't created in this run then a BackupException will be raised.
Remember to call restoreFiles to rollback the changes at cleanup.
@param fileP: The absolute path to the file that is to be backed up.
@param suffix: The suffix used for marking the backup file.
@param copy: if True then the file will be copied and otherwise it will be moved.
'''
if fileP not in cls.files:
if not os.path.isfile(fileP):
raise BackupSourceException("File '%s' does not exist or is not accessible." % fileP)
backup = "%s%s" % (fileP, suffix)
if os.path.isfile(backup) and cls.ignore is False:
raise BackupExistsException("Backup file '%s' already exists." % backup)
if copy:
shutil.copy2(fileP, backup)
logging.debug("Backed up '%s' (Copy)" % fileP)
else:
shutil.move(fileP, backup)
logging.debug("Backed up '%s' (Move)" % fileP)
cls.files.add(fileP)
return True
else:
logging.info("File '%s' already backed up in this session. Skipping." % fileP)
return False
def initStart(cls, proc, autoStop = True, expected = "is running.\n", service = True):
'''
Used for starting a process which has a startup file in the init.d directory.
If the process return code was different than 0 then an InitException will be raised.
If expected is passed and the process isn't running then an InitException will be raised.
@param proc: the name of the startup script in the init.d directory.
@param autoStop: if True then the process will be stopped when stopStartedInits is called.
@param expected: the string that is expected to end the status string from an init script in it's running state. If None is passed then the status won't be checked.
'''
if service:
procFull = "service %s" % proc
else:
procFull = "/etc/init.d/%s" % proc
cls.initStop(proc, verbose = False)
ret = Console.call("%s %s" % (procFull, "start"))
if ret[0] != 0:
logging.error(ret)
raise InitException("Couldn't start init")
if expected is not None:
if not cls.initStatus(proc, expected):
raise InitException("%s didn't start" % proc)
if autoStop:
cls.inits.append((proc, expected))
logging.info("Init '%s' started" % proc)
def __init__(self, filePath, port, logTo = "/var/log/hsn2"):
'''
Sets up a SimpleHTTPServer with a root at the given filePath.
If the website root doesn't exist then a SetupException will be raised.
If a website was already started in this run on the port with a different filePath then a SetupException will be raised.
Repeated calls with the same port and filePath will be ignored (but logged).
@param filePath: The root of the website.
@param port: The port on which the website is to be available.
@param logTo: Where to store the access log file for the site (named site-<PORT>.log).
'''
if not os.path.isdir(filePath):
raise SetupException("Specified path is not a accessible directory %s" % (filePath))
if (self.__class__.sites.get(port) is None):
self.proc = subprocess.Popen(["python", "-m", "SimpleHTTPServer", "%d" % port],
stdout = open(os.path.join(logTo, "site-%d.log" % port), "w"),
stderr = subprocess.STDOUT,
cwd = filePath)
self.filePath = filePath
self.port = port
self.__class__.sites[port] = self
logging.info("Setup website with root at '%s on port %d" % (filePath, port))
else:
filePath2 = self.__class__.sites[port].getFilePath()
if (filePath != filePath2):
raise SetupException("A site using the port %d has already been setup with it's root at %s" % (port, filePath2))
else:
logging.info("Site on port %d with root at %s already running. Skipping setup." % (port, filePath2))
def getFromCouch(jobId, objectId, serviceName = "", verbose = True, url = "http://localhost:5984/hsn", timeout = 10): ''' Wrapper for getting objects from couchDB. @param jobId: The job that the data belongs to. @param objectId: The objectId that the data belongs to. @param serviceName: The name of the service that was reported (reporter configuration). @param url: The location of the couch database REST api. @param timeout: Number of seconds to wait for connection. @return: dictionary with the data if the object was found and otherwise None ''' if len(serviceName) > 0: serviceName = ":%s" % serviceName objUrl = "%s/%d:%d%s" % (url, long(jobId), long(objectId), serviceName) objUrl = urllib.quote(objUrl, safe = ":/") try: handle = getUrlHandle(objUrl, None, "GET") except urllib2.URLError as e: logging.error(e) return None import json text = ''.join(handle.readlines()) if verbose: logging.info(text) dictionary = json.loads(text) return dictionary
def shutdownWebsite(self):
'''
Shutdown a website which was started in this run on the given port.
'''
logging.info("Shutting down website on port %d" % self.getPort())
self.proc.terminate()
return True
def aggregateFile(self, filePath):
'''
Collect chosen files into a separate directory.
@param filePath: the file to aggregate
'''
split = os.path.split(filePath)
shutil.move(filePath, os.path.join(self.putTo, split[1]))
logging.info("Aggregated %s to %s" % (filePath, self.putTo))
def call(cls, string, stderr = False):
'''
Wrapper for handling simple synchronous calls to external binaries. Calls sysCall.
@param string: The string that will be split into arguments for the call.
@param stderr: Whether to capture stderr.
@return: tuple (return value, STDOUT, STDERR) for the process that was called.
'''
logging.info("Calling: %s" % string)
return cls.sysCall(string.split(), stderr = stderr)
def setWorkflowsFolder(cls, path):
'''
Change the workflow directory setting in the Framework configuration file.
@param path: Directory containing workflows to add.
'''
if os.path.isdir(path):
logging.info("Searching for .hwl files in %s" % path)
for f in os.listdir(path):
if re.search('\.hwl$', f) is not None:
cls.setWorkflow(os.path.join(path, f))
def setWorkflow(cls, path, name = None):
'''
Change the workflow directory setting in the Framework configuration file.
@param path: Full path to .hwl workflow file.
@param name: Optional name for the workflow.
'''
if os.path.isfile(path):
logging.info("Updating new workflow: %s" % path)
if name is None:
name = cls.getWorkflowName(path)
Console.call("hc w u -f %s %s" % (path, name))
def initStop(cls, proc, expected = None, verbose = True):
'''
Used for stopping a process which has a startup file in the init.d directory.
If expected is passed and the process isn't running then an InitException will be raised.
@param proc: the name of the startup script in the init.d directory (or an absolute path).
@param expected: the string that is expected to end the status string from an init script in it's running state. If None is passed then the status won't be checked.
'''
procFull = "service %s" % proc
if expected is not None:
if not cls.initStatus(proc, expected):
raise InitException("%s was dead before calling stop." % proc)
Console.call("%s %s" % (procFull, "stop"))
if verbose:
logging.info("Init '%s' stopped" % proc)
def deleteCouchDB(cls, url = "http://localhost:5984", database = "hsn"):
'''
Deletes the specified couch database.
@param url: location of the couchdb instance
@param database: the name of the database that is to be cleared.
'''
try:
deletion = getUrlHandle("%s/%s" % (url, database), method = "DELETE").readlines()
logging.info("Deleting couchdb database: %s/%s - %s" % (url, database, ''.join(deletion).strip()))
except urllib2.HTTPError as e:
if e.code == 404:
logging.info("Couchdb database not found at: %s/%s" % (url, database))
else:
raise e
def submitJob(cls, params, verbose = True): ''' Helper for submitting jobs and retrieving their ids. @param params: The workflow name and parameters that are to be submitted with the job. @param verbose: Whether the response is to appear in the logging output. @return: jobId if found a job id and otherwise None. ''' params = "hc j s " + params ret = cls.call(params) if verbose: logging.info(ret[1]) result = cls.jobIdRegexp.search(ret[1]) if result is None: return None else: return result.group(1)
def aggregate(self, takeFrom = "/var/log/hsn2", suffix = ".log"):
'''
Collect chosen files into a separate directory.
@param takeFrom: where files will be taken from
@param suffix: The suffix of the files that are to be aggregated. Passing None means all files.
'''
takeFrom = os.path.abspath(takeFrom)
if takeFrom == self.putTo:
logging.warning("takeFrom is the same as putTo '%s'. Aggregation skipped!" % takeFrom)
else:
logFiles = os.listdir(takeFrom)
for logFile in logFiles:
if suffix is None or os.path.splitext(logFile)[1] == suffix:
shutil.move(os.path.join(takeFrom, logFile), os.path.join(self.putTo, logFile))
logging.info("Aggregated logs from %s to %s" % (takeFrom, self.putTo))
def setServices(cls, serviceList, confFile = "/etc/hsn2/framework.conf"):
'''
Change the workflow directory setting in the Framework configuration file.
@param path: The new value of the workflow directory setting.
@param confFile: Path to the configuration file.
'''
if os.path.isfile(confFile):
Backup.backupFile(confFile)
serviceList = ', '.join(serviceList)
logging.info("Setting services to %s" % serviceList)
conf = cls.getConf(confFile)
pattern = re.compile(r"AMQP.services=.*")
for i in range(len(conf)):
match = pattern.match(conf[i])
if match is not None:
conf[i] = pattern.sub("AMQP.services=%s" % serviceList, conf[i], 1)
break
cls.setConf(confFile, conf)
def waitForCompletion(cls, jobId = 1, maxTime = 20, period = 2, verbose = False): ''' Waits for console to report job as completed/aborted. @param id: The id of the job that the function will be waiting for. @param maxTime: The maximum amount of time to wait. If set to a negative number then keeps waiting. @param period: ''' jobId = int(jobId) command = "hc j d %d" % jobId timeStart = time.time() timeNow = time.time() while maxTime < 0 or (timeNow - timeStart < maxTime): timeNow = time.time() time.sleep(period) output = cls.call(command) output = output[1] if verbose: logging.info(output) if cls.completedRegexp.search(output): return output if cls.abortedRegexp.search(output): return False return False
def initStatus(cls, proc, expected = "is running.\n", pid = None):
'''
Used for checking if a process that was started with it's init.d startup file is running.
@param proc: the name of the startup script in the init.d directory (or an absolute path).
@param expected: the string that is expected to end the status string from an init script in it's running state.
@return: True if the expected string was found at the end of the status message and otherwise False.
'''
procFull = "service %s" % proc
expLen = len(expected)
timeout = 60
keep_running = True
start = time.time()
while keep_running:
ret = Console.call("%s %s" % (procFull, "status"), True)
if ret[0] == 0 or time.time() - start > timeout:
keep_running = False
else:
time.sleep(1)
if ret[1][-expLen:] != expected:
logging.info("Got:'%s', Expected:'%s'" % (ret[1][-expLen:], expected))
return False
return True
def checkStatus(cls, status, jobid, url, timeout = 90):
'''
Checks if the status of the url processed by capture-hpc appeared in log file.
@param status: string, a status to ask for
@param jobid: string, expected job id
@param url: string, url processed by capture-hpc
@param timeout: int, a number of seconds to wait for status to show up
'''
time_started = time.time()
timeout_time = time_started + timeout
logging.info("Checking log for: %s %s %s" % (status, jobid, url))
while "%s %s %s" % (status, jobid, url) not in cls.getLog() and time.time() < timeout_time:
time.sleep(1)
time_finished = time.time()
if time_finished < timeout_time:
logging.info("Finished after %d seconds." % (time_finished - time_started))
return True
else:
logging.info("%s %s %s not added to %s in expected time (less than %d seconds)" % (status, jobid, url, cls.log, timeout))
return False
def toprettyxml(self):
return self.dom.toprettyxml()
if __name__ == "__main__":
setupLogging(logPath = None, logToStream = True, logLevel = logging.INFO)
Configuration.setHSN2TestPath("/tmp/tests")
try:
Backup.backupOldLogs()
Backup.backupCouchDB("hsn", exception = False)
agg = LogAggregator(putTo = "/var/log/hsn2/c")
except (BackupException, IOError, OSError) as e:
Backup.restoreFiles()
raise e
setupLogging(logPath = "/var/log/hsn2/test.log", logToStream = True, logLevel = logging.DEBUG)
logging.info("Testing commons")
try:
Configuration.resetJobCounter()
Configuration.setConsoleConf("127.0.0.1", 5672, 4)
Configuration.setCouchConf()
Starter.isRunning("/var/run/couchdb/couchdb.pid")
Configuration.setWorkflowsFolder("/tmp/tests/workflows/a")
Website("/tmp/tests", 81)
Website("/var/log", 82)
Website("/var/log", 82)
Starter.initCouchDB()
Starter.initRabbitMQ()
Starter.initStart("hsn2-framework")
Starter.initStart("hsn2-object-store-mongodb")
Starter.initStart("hsn2-data-store")
Starter.initStart("hsn2-swf-cve")
