def __init__( self, path ):
self._steps = []
config = ConfigParser.SafeConfigParser()
config.read( path )
self._path = path
self._platforms = []
found = True
stepNum = 0
while found:
sec = "Step{0}".format( stepNum )
if not config.has_section( sec ):
found = False
continue
stepNum += 1
installscript = utils.makeAbsolutePath( utils.get_config_option( config, None, "installscript", None, sec ), os.path.dirname( path ) )
checkerTestDir = utils.get_config_option( config, None, "checkerTestDir", None, sec )
checkerTestDir = utils.makeAbsolutePath( checkerTestDir, os.path.dirname( path ) ) if checkerTestDir else ""
timeout = int( utils.get_config_option( config, None, "timeout", 60 * 60, sec ) )
self._steps.append( Step( installscript, checkerTestDir, timeout ) )
self._name = utils.get_config_option( config, None, "name", utils.basename( path ) )
self._targetDirectory = utils.get_config_option( config, None, "targetDirectory", "" )
self._maintenanceToolLocation = utils.get_config_option( config, None, "maintenanceToolLocation", "" )
platforms = utils.get_config_option( config, None, "platforms" )
if platforms != None:
self._platforms = platforms.split( ',' )
def fromVMRunAndPath( vmrun, path ):
config = ConfigParser.SafeConfigParser()
config.read( path )
hostType = utils.get_config_option( config, None, "type", "", "Host" )
hostLocation = utils.get_config_option( config, None, "location", "", "Host" )
hostUsername = utils.get_config_option( config, None, "username", "", "Host" )
hostPassword = utils.get_config_option( config, None, "password", "", "Host" )
vmxVal = utils.get_config_option( config, None, "vmx" )
if not len( hostType ) == 0:
vmx = utils.makeAbsolutePath( vmxVal, os.path.dirname( path ) )
else:
vmx = vmxVal
username = utils.get_config_option( config, None, "username", "nokia" )
password = utils.get_config_option( config, None, "password", "nokia" )
tempDir = utils.get_config_option( config, None, "tempDir", "c:\\windows\\temp" )
ostype = utils.get_config_option( config, None, "os", "windows" )
vm = VirtualMachine( vmrun, vmx, username, password, tempDir, ostype )
vm._name = utils.get_config_option( config, None, "name", utils.basename( path ) )
vm._snapshot = utils.get_config_option( config, None, "snapshot", "base" )
vm._python = utils.get_config_option( config, None, "python", "c:/python26/python.exe" )
vm._hostType = hostType
vm._hostLocation = hostLocation
vm._hostUsername = hostUsername
vm._hostPassword = hostPassword
return vm
def __init__(self, path):
self._steps = []
config = ConfigParser.SafeConfigParser()
config.read(path)
self._path = path
self._platforms = []
found = True
stepNum = 0
while found:
sec = "Step{0}".format(stepNum)
if not config.has_section(sec):
found = False
continue
stepNum += 1
installscript = utils.makeAbsolutePath(
utils.get_config_option(config, None, "installscript", None,
sec), os.path.dirname(path))
checkerTestDir = utils.get_config_option(config, None,
"checkerTestDir", None,
sec)
checkerTestDir = utils.makeAbsolutePath(
checkerTestDir,
os.path.dirname(path)) if checkerTestDir else ""
timeout = int(
utils.get_config_option(config, None, "timeout", 60 * 60, sec))
self._steps.append(Step(installscript, checkerTestDir, timeout))
self._name = utils.get_config_option(config, None, "name",
utils.basename(path))
self._targetDirectory = utils.get_config_option(
config, None, "targetDirectory", "")
self._maintenanceToolLocation = utils.get_config_option(
config, None, "maintenanceToolLocation", "")
platforms = utils.get_config_option(config, None, "platforms")
if platforms != None:
self._platforms = platforms.split(',')
def makePlot1cFilelists(golddir, resultdir, outputdir, genefile, queryNames):
golddir = utils.makeAbsolutePath(golddir)
resultdir = utils.makeAbsolutePath(resultdir)
outputdir = utils.makeAbsolutePath(outputdir)
genefile = utils.makeAbsolutePath(genefile)
queryfilelist = os.path.join(outputdir, 'filelist_seek1c.query')
goldfilelist = os.path.join(outputdir, 'filelist_seek1c.gold')
gscorefilelist = os.path.join(outputdir, 'filelist_seek1c.gscore')
includefilelist = os.path.join(outputdir, 'filelist_seek1c.include')
excludefilelist = os.path.join(outputdir, 'filelist_seek1c.exclude')
filecount = 0
queryNames = [name.rstrip() for name in queryNames.split(',')]
for queryName in queryNames:
queryDir = os.path.join(golddir, queryName)
gscoreDir = os.path.join(resultdir, queryName)
filepattern = os.path.join(queryDir, r'[0-9]*.query')
for queryfile in glob.iglob(filepattern):
filecount += 1
# Add a line to each filelist
utils.file_appendline(queryfilelist, queryfile)
utils.file_appendline(goldfilelist,
re.sub('query$', 'gold', queryfile))
utils.file_appendline(includefilelist, genefile)
utils.file_appendline(excludefilelist, queryfile)
# gscore file is in the result dir rather than gold dir
qbasename = os.path.basename(queryfile)
gscorefile = os.path.join(gscoreDir,
qbasename.replace('query', 'gscore'))
utils.file_appendline(gscorefilelist, gscorefile)
if filecount == 0:
print("No matching files found")
exit(-1)
print("Num files processed: {}".format(filecount))
import ConfigParser, datetime, optparse, os, sys
from functools import partial
import cdashreporter, control, ftpsource, source, reporter, utils
def die( msg ):
sys.stderr( msg + '\n' )
sys.exit( 1 )
optionParser = optparse.OptionParser(usage="%prog [options] configfile installer0 [installer1 ...]", version="%prog 0.1")
optionParser.add_option("-r", "--vmrun", dest="vmrun", help="vmrun executable to use", metavar="VMRUN" )
optionParser.add_option("-s", "--only-since", dest="since", help="test only installers newer than timestamp (YYYY-MM-DD or YYYY-MM-DD-hh-mm)", metavar="SINCE" )
optionParser.add_option("-c", "--checkerInstallation", dest="checkerInstallation", help="checker installation to use for post-installation checks", metavar="CHECKERINSTALLATION" )
(options, args) = optionParser.parse_args()
try:
configpath = utils.makeAbsolutePath( args[0], os.getcwd() )
except IndexError:
optionParser.print_usage( sys.stderr )
sys.exit( 1 )
config = ConfigParser.SafeConfigParser()
config.read( configpath )
#make an unary functor to create absolute paths
abspath = partial( utils.makeAbsolutePath, relativeTo=os.path.dirname( configpath ) )
vmrun = utils.get_config_option( config, options.vmrun, "vmrun", utils.findVMRun() )
useGui = utils.get_config_option( config, None, "gui", "true" ).lower() == "true"
createErrorSnapshots = utils.get_config_option( config, None, "createErrorSnapshots", "true" ).lower() == "true"
hostType = utils.get_config_option( config, None, "type", "", "Host" )
hostLocation = utils.get_config_option( config, None, "location", "", "Host" )
def create_queries(groups,
query_count,
max_query_genes,
outBaseFilename,
query_pct=None,
max_groups=None):
"""
Create queries out of groups of biologically informative gene sets.
Each group is a set of genes that are correlated with each other.
One query line will be made for each query_count in the list, the query_count
indicates what how many of the gene group to use as the query, the rest
will be the target result genes.
"""
outBaseFilename = makeAbsolutePath(outBaseFilename)
if os.path.basename(outBaseFilename) in ['', None]:
print('output basename is empty')
return
print("output to {}".format(outBaseFilename + '.query.txt'))
queryFH = open(outBaseFilename + '.query.txt', 'w')
goldStdFH = open(outBaseFilename + '.goldStd.txt', 'w')
groupFH = open(outBaseFilename + '.group.txt', 'w')
num_groups = 0
for group in groups:
if max_groups is not None:
if num_groups >= max_groups:
break
num_groups += 1
# if query_pct specified, translate that into the number of genes for each query
if query_pct is not None and len(query_pct) > 0:
num_query_genes = [(group.size * qpct / 100.0)
for qpct in query_pct]
if max(num_query_genes) > max_query_genes:
# Using the max(query_pct), calculate a group.size that will keep the
# number of query genes below max_query_genes
max_qpct = max(query_pct)
gsize = max_query_genes * 100.0 / max_qpct
num_query_genes = [(gsize * qpct / 100.0)
for qpct in query_pct]
# Append the counts derived from pct to the count array
query_count.extend(num_query_genes)
group.queries = []
for qnum in query_count:
query = StructDict()
if qnum < 10:
qnum = math.ceil(qnum)
else:
qnum = math.floor(qnum)
assert qnum <= max_query_genes
assert qnum <= group.size - 1
query.qnum = qnum
# create a random list of index values for the query genes
query.q_indicies = random.sample(range(group.size), qnum)
# create the inverse list for the results genes
query.res_indicies = list(
set(range(group.size)) - set(query.q_indicies))
assert len(query.q_indicies) == query.qnum
assert len(query.res_indicies) == group.size - query.qnum
assert set(query.q_indicies).isdisjoint(set(query.res_indicies))
assert len(group.genes) == group.size
assert max(query.q_indicies) < group.size
assert max(query.res_indicies) < group.size
# select the query genes and result genes
query.genes = [group.genes[i] for i in query.q_indicies]
query.results = [group.genes[i] for i in query.res_indicies]
# write the query genes to a file
queryFH.write(" ".join(query.genes) + "\n")
# write the result genes to a file
goldStdFH.write(" ".join(query.results) + "\n")
# write the group to a file
groupFH.write("\t".join([group.id, group.desc]) + "\n")
# groupFH.write("\t".join(group.genes) + "\n")
group.queries.append(query)
# end for query percents
# end for groups
queryFH.close()
goldStdFH.close()
groupFH.close()
