def runServer(self):
"""
Run a redis-server. This function launches a separate python process that manages the server.
Writing to the special key "TERMINATE" signals this thread to safely shut
down the DB and save the results. After finishing, the data will
eventually be written to snapshotFile.
Returns a tuple containing an updated version of the database config dbElem and the
path to the log file.
"""
self.databaseDir = self.fileStore.getLocalTempDir()
# log file can be saved in a subdirectory of where the snapshot is being saved
self.logPath = os.path.join(self.databaseDir, "redis.log")
open(self.logPath, 'a').close()
self.dbElem.setDbHost(getHostName())
# Find a suitable port to run on.
try:
occupiedPorts = findOccupiedPorts()
unoccupiedPorts = set(range(1025, MAX_REDIS_PORT)) - occupiedPorts
port = random.choice(list(unoccupiedPorts))
except:
logger.warning(
"Can't find which ports are occupied--likely netstat is not installed."
" Choosing a random port to start the DB on, good luck!")
port = random.randint(1025, MAX_REDIS_PORT)
self.dbElem.setDbPort(port)
try:
cactus_call(shell=False, parameters=['redis-server', '--version'])
except:
raise RuntimeError("redis-server is not installed")
process = RedisServerProcess(self)
process.daemon = True
process.start()
if not self.blockUntilServerIsRunning():
try:
with open(self.logPath) as f:
log = f.read()
except:
log = ''
raise RuntimeError(
"Unable to launch redis-server in time. Log: %s" % log)
return process, self.dbElem, self.logPath
def runKtserver(dbElem,
fileStore,
existingSnapshotID=None,
snapshotExportID=None):
"""
Run a KTServer. This function launches a separate python process that manages the server.
Writing to the special key "TERMINATE" signals this thread to safely shut
down the DB and save the results. After finishing, the data will
eventually be written to snapshotFile.
Returns a tuple containing an updated version of the database config dbElem and the
path to the log file.
"""
logPath = fileStore.getLocalTempFile()
dbElem.setDbHost(getHostName())
# Find a suitable port to run on.
try:
occupiedPorts = findOccupiedPorts()
unoccupiedPorts = set(xrange(1025, MAX_KTSERVER_PORT)) - occupiedPorts
port = random.choice(list(unoccupiedPorts))
except:
logger.warning(
"Can't find which ports are occupied--likely netstat is not installed."
" Choosing a random port to start the DB on, good luck!")
port = random.randint(1025, MAX_KTSERVER_PORT)
dbElem.setDbPort(port)
process = ServerProcess(dbElem, logPath, fileStore, existingSnapshotID,
snapshotExportID)
process.daemon = True
process.start()
if not blockUntilKtserverIsRunning(logPath):
try:
with open(logPath) as f:
log = f.read()
except:
log = ''
raise RuntimeError("Unable to launch ktserver in time. Log: %s" % log)
return process, dbElem, logPath
def runKtserver(dbElem, fileStore, existingSnapshotID=None, snapshotExportID=None):
"""
Run a KTServer. This function launches a separate python process that manages the server.
Writing to the special key "TERMINATE" signals this thread to safely shut
down the DB and save the results. After finishing, the data will
eventually be written to snapshotFile.
Returns a tuple containing an updated version of the database config dbElem and the
path to the log file.
"""
logPath = fileStore.getLocalTempFile()
dbElem.setDbHost(getHostName())
# Find a suitable port to run on.
try:
occupiedPorts = findOccupiedPorts()
unoccupiedPorts = set(xrange(1025,MAX_KTSERVER_PORT)) - occupiedPorts
port = random.choice(list(unoccupiedPorts))
except:
logger.warning("Can't find which ports are occupied--likely netstat is not installed."
" Choosing a random port to start the DB on, good luck!")
port = random.randint(1025,MAX_KTSERVER_PORT)
dbElem.setDbPort(port)
process = ServerProcess(dbElem, logPath, fileStore, existingSnapshotID, snapshotExportID)
process.daemon = True
process.start()
if not blockUntilKtserverIsRunning(logPath):
try:
with open(logPath) as f:
log = f.read()
except:
log = ''
raise RuntimeError("Unable to launch ktserver in time. Log: %s" % log)
return process, dbElem, logPath
def cactusPrepare(options, project):
""" annotate a SeqFile with ancestral names as well as paths for output sequences."""
# read the input
seqFile = SeqFile(options.seqFile)
configNode = ET.parse(options.configFile).getroot()
config = ConfigWrapper(configNode)
if not options.wdl and not options.toil:
# prepare output sequence directory
# todo: support remote (ie s3) output directory
try:
os.makedirs(options.outDir)
except:
pass
if not os.path.isdir(options.outDir):
raise RuntimeError('Unable to create output sequence directory \'{}\''.format(options.outDir))
if not os.access(options.outDir, os.W_OK):
logger.warning('Output sequence directory is not writeable: \'{}\''.format(options.outDir))
if options.preprocessOnly or options.gpu:
if options.preprocessOnly:
# hack the configfile to skip preprocessing and write it to the output dir
config.removePreprocessors()
if options.gpu:
# hack the configfile to toggle on gpu lastz
cafNode = findRequiredNode(config.xmlRoot, "caf")
cafNode.attrib["gpuLastz"] = "true"
# realigning doesn't mix well with lastz so we make sure it's off
# https://github.com/ComparativeGenomicsToolkit/cactus/issues/271
cafNode.attrib["realign"] = "0"
options.configFile = os.path.join(options.outDir, 'config-prepared.xml')
sys.stderr.write("configuration saved in {}\n".format(options.configFile))
config.writeXML(options.configFile)
# pass through the config file to the options
# todo (don't like second hard-code check of .xml path)
if options.configFile != os.path.join(cactusRootPath(), "cactus_progressive_config.xml") and not options.wdl:
options.cactusOptions += ' --configFile {}'.format(options.configFile)
# get the ancestor names
tree = MultiCactusTree(seqFile.tree)
tree.nameUnlabeledInternalNodes(prefix = config.getDefaultInternalNodePrefix())
# make the output
outSeqFile = SeqFile()
outSeqFile.tree= tree
outSeqFile.pathMap = copy.deepcopy(seqFile.pathMap)
outSeqFile.outgroups = copy.deepcopy(seqFile.outgroups)
# update paths for preprocessed leaves or inferred ancestors
for node in outSeqFile.tree.breadthFirstTraversal():
name = outSeqFile.tree.getName(node)
leaf = outSeqFile.tree.isLeaf(node)
if leaf or (not leaf and name not in seqFile.pathMap and not options.preprocessOnly):
out_basename = seqFile.pathMap[name] if name in seqFile.pathMap else '{}.fa'.format(name)
outSeqFile.pathMap[name] = os.path.join(options.outDir, os.path.basename(out_basename))
if options.wdl:
# uniquify name in wdl to prevent collisions
outSeqFile.pathMap[name] += '.pp'
# write the output
if options.outSeqFile:
with open(options.outSeqFile, 'w') as out_sf:
out_sf.write(str(outSeqFile))
# write the instructions
if options.toil:
with Toil(options) as toil:
if options.restart:
toil.restart()
else:
get_plan(options, project, seqFile, outSeqFile, toil=toil)
else:
print(get_plan(options, project, seqFile, outSeqFile, toil=None))