def __init__(self):
self.logf = logging.getLogger('Main')
self.configPath = 'config/' + platform.system() + '.conf'
self.initPlatformConfig()
self.builder = Builder(self.config)
self.q2ded = quake2.Server(self.config)
#TODO self.quake2 = quake2.Client(self.config)
self.running = True
def runPackage(self):
testType = self.getConfigSetting("base/type", "standalone")
self.buildStatus = {}
if testType == "remote" and "testRun" in self.configuration:
jUrl = self.getConfigSetting("testRun/host", "")
jUrl += self.getConfigSetting("testRun/qxPath", "")
jUrl += "buildStatus.json"
self.buildStatus = util.getJsonFromUrl(jUrl)
if testType == "local":
try:
buildStatusFile = os.path.join(self.config["build"]["stageDir"], "trunk", "qooxdoo", "buildStatus.json")
self.buildStatus = Builder.getBuildStatusFromFile(buildStatusFile)
except Exception, e:
pass
def test_config(self):
with self.assertRaises(IOError):
bc = BuildConfig("/does/not/exist")
with tmpd():
with open("config", "w") as f:
f.write(example_config())
work_dir = tempfile.mkdtemp()
work_dir = "/tmp/builder-work"
config = BuildConfig("config")
builder = Builder(work_dir, config)
builder.update_source()
builder.build()
from build import Builder
b = Builder()
b.addfile('example/tank/crate.js')
b.addfile('example/tank/tank.js')
b.addfile('example/tank/circle.js')
b.addMainScene('example/tank/main.js', 'Main')
b.addResources('example/tank/tank.png')
b.build()
self.log.error("Error while getting remote build status: " + str(e))
if testType == "local":
try:
buildStatusFile = os.path.join(self.config["build"]["stageDir"], "trunk", "qooxdoo", "buildStatus.json")
self.buildStatus = Builder.getBuildStatusFromFile(buildStatusFile)
except Exception, e:
pass
if "lint" in self.configuration:
self.runLint(self.configuration["lint"])
if "build" in self.configuration:
buildConfig = self.configuration["build"]
#buildConfig["buildLogDir"] = self.getConfigSetting("base/logDirectory", "") + "/" + buildConfig["buildLogDir"]
builder = Builder(buildConfig, self.log)
builder.buildAll()
self.buildStatus = builder.buildStatus
if "generator" in self.configuration:
self.log.info("Starting Generator test run")
self.log.indent()
generatorResults = Builder.testGenerator(self.configuration["generator"], self.log)
reportServer = self.getConfigSetting("reporting/reportServer", False)
if reportServer:
self.log.info("Sending Generator test results to report server")
response = reporting.sendResultToReportServer(reportServer, generatorResults, "generatorRun")
self.log.info("Report Server response: %s" %response)
self.log.outdent()
if not "testRun" in self.configuration:
#!/usr/bin/env python3
import sys
import os
import shutil
from build import Builder
package_name = Builder.package_name_from_filename(__file__)
dependencies = ()
def prepare(builder):
archive_name = package_name + '.tar.gz'
builder.extract(archive_name)
return True
def build(builder):
os.chdir(package_name)
os.mkdir('build', 0o750)
os.chdir('build')
builder.cmake(build_type='Release', install=True)
if builder.host_system == 'windows':
# The install target places libs in a Release sub-folder on Windows.
for filename in (builder.install_prefix / 'lib' / 'Release').glob('*'):
shutil.move(filename,
builder.install_prefix / 'lib' / filename.name)
shutil.rmtree(builder.install_prefix / 'lib' / 'Release')
# Remove license file which gets installed directly to PREFIX path on Windows.
class TestRun:
def __init__(self, config, simulate=False, logger=log):
self.configuration = config
if self.getConfigSetting("base/type", "") == "standalone-generator":
self.configuration = self.getConfigFromGenerator()
self.log = logger
self.simulate = simulate
self.startDate = util.getTimestamp()
self.buildStatus = {}
if "base" in self.configuration:
self.configuration["base"]["testHostName"] = util.getHostName()
self.log.info("New test run started on %s" % self.startDate)
def getConfigSetting(self, path="", default=False, config=False):
if not config:
config = self.configuration
pathParts = path.split("/")
dict = config
while dict:
for keyName in pathParts:
# invalid node name
if not keyName in dict:
return default
# final search path node, so we've found the right setting
if keyName == pathParts[len(pathParts) - 1]:
return dict[keyName]
# haven't reached the final step yet, but the current value is no dict so
# we've nowhere else to look
if type(dict[keyName]).__name__ != "dict":
return default
# keep looking
dict = dict[keyName]
return default
def getConfigFromGenerator(self):
jobConfig = self.configuration
seleniumPath = os.path.abspath(jobConfig["selenium-java-client"])
testConfig = {
"base": {
"type": "standalone",
},
"selenium": {
"seleniumHost": jobConfig["selenium-host"],
"seleniumPort": jobConfig["selenium-port"],
"rhinoJar": jobConfig["rhino"],
"seleniumDir": os.path.dirname(seleniumPath),
"seleniumClientDriverJar": os.path.basename(seleniumPath),
},
"browsers": {
"Browser": jobConfig["browser"]
},
"testRun": {
"runType": "",
"simulatorDirectory": jobConfig["simulator"],
"host": jobConfig["aut-host"],
"applications": {}
}
}
autName = jobConfig["aut-name"]
testConfig["testRun"]["applications"][autName] = {
"path": jobConfig["aut-path"],
"simulationScript": jobConfig["simulator"] + "/simpleTest.js",
"processLog": True,
"browsers": [{
"browserId": "Browser"
}]
}
simulationOptions = ["logger=console"]
simulationOptions.append("testClasspath=" +
jobConfig["test-classpath"])
testClasses = ",".join(jobConfig["test-include"])
simulationOptions.append("testClasses=" + testClasses)
if "simulation-options" in jobConfig:
simulationOptions.extend(jobConfig["simulation-options"])
testConfig["testRun"]["applications"][autName][
"simulationOptions"] = simulationOptions
if "java-classpath-separator" in jobConfig:
testConfig["base"]["classPathSeparator"] = jobConfig[
"java-classpath-separator"]
return testConfig
def runPackage(self):
self.log.indent()
testType = self.getConfigSetting("base/type", "standalone")
if testType == "remote" and "testRun" in self.configuration:
jUrl = self.getConfigSetting("testRun/host", "")
jUrl += self.getConfigSetting("testRun/qxPath", "")
jUrl += "buildStatus.json"
try:
self.buildStatus = util.getJsonFromUrl(jUrl)
except Exception, e:
self.log.error("Error while getting remote build status: " +
str(e))
if testType == "local":
try:
buildStatusFile = os.path.join(
self.config["build"]["stageDir"], "trunk", "qooxdoo",
"buildStatus.json")
self.buildStatus = Builder.getBuildStatusFromFile(
buildStatusFile)
except Exception, e:
pass
try:
buildStatusFile = os.path.join(
self.config["build"]["stageDir"], "trunk", "qooxdoo",
"buildStatus.json")
self.buildStatus = Builder.getBuildStatusFromFile(
buildStatusFile)
except Exception, e:
pass
if "lint" in self.configuration:
self.runLint(self.configuration["lint"])
if "build" in self.configuration:
buildConfig = self.configuration["build"]
#buildConfig["buildLogDir"] = self.getConfigSetting("base/logDirectory", "") + "/" + buildConfig["buildLogDir"]
builder = Builder(buildConfig, self.log)
builder.buildAll()
self.buildStatus = builder.buildStatus
if "generator" in self.configuration:
self.log.info("Starting Generator test run")
self.log.indent()
generatorResults = Builder.testGenerator(
self.configuration["generator"], self.log)
reportServer = self.getConfigSetting("reporting/reportServer",
False)
if reportServer:
self.log.info(
"Sending Generator test results to report server")
response = reporting.sendResultToReportServer(
reportServer, generatorResults, "generatorRun")
def __init__(self):
ui.widget(
"Label",
":larchify",
text=" *** <strong>%s</strong> ***" % _("The system to be compressed must be installed and ready."),
)
ui.widget(
"Button",
"^:locales",
text=_("Edit supported locales"),
tt=_("Edit the /etc/locale.gen file to select supported glibc locales"),
)
ui.widget(
"Button",
"^:rcconf",
text=_("Edit Arch configuration file (/etc/rc.conf)"),
tt=_("Edit the /etc/rc.conf file to configure the live system"),
)
ui.widget("OptionalFrame", ":larchify_advanced", text=_("Advanced Options"))
ui.widget(
"Button",
"^:initcpio",
text=_("Edit mkinitcpio.conf"),
tt=_("Edit the configuration file for generating the initramfs via mkinitcpio"),
)
ui.widget(
"Button",
"^:overlay",
text=_("Edit overlay (open in file browser)"),
tt=_("Open a file browser on the profile's 'rootoverlay'"),
)
ui.widget("CheckBox", "^:ssh", text=_("Generate ssh keys"), tt=_("Enables pre-generation of ssh keys"))
ui.widget(
"CheckBox",
":oldsquash",
text=_("Reuse existing system.sqf"),
tt=_("Reuse existing system.sqf, to save time if the base system hasn't changed"),
)
ui.widget("Button", "^&-:build", text=_("Larchify"), tt=_("Build the main components of the larch system"))
ui.widget("Frame", ":users", text=_("User accounts"))
ui.widget(
"List",
"^&-:utable",
selectionmode="Single",
tt=_("Click on a row to select, click on a selected cell to edit"),
)
ui.widget("Button", "^&-:useradd", text=_("Add user"), tt=_("Create a new user-name"))
ui.widget("Button", "^&:userdel", text=_("Delete user"), tt=_("Remove the selected user-name"))
ui.layout(
":page_larchify",
[
"*VBOX*",
":larchify",
":users",
"*SPACE",
["*HBOX*", ":locales", "*SPACE", ":rcconf"],
"*SPACE",
":larchify_advanced",
["*HBOX*", "*SPACE", "&-:build"],
],
)
ui.layout(
":larchify_advanced",
["*HBOX*", ["*VBOX*", ":initcpio", ":overlay"], "*SPACE", ["*VBOX*", ":ssh", "*SPACE", ":oldsquash"]],
)
ui.layout(":users", ["*VBOX*", "&-:utable", ["*HBOX*", "&-:useradd", "&:userdel", "*SPACE"]])
self.userheaders = [
_("User-Name"),
_("Password"),
_("Group"),
"UID",
_("'skel' directory"),
_("Additional Groups"),
_("Expert options"),
]
ui.command("&-:utable.setHeaders", self.userheaders)
ui.command("&-:utable.compact")
ui.sendui("^&-:utable clicked")
self.builder = Builder()
self.sshgen = True
class BuildPage:
"""This class manages the page dealing with larch system building.
"""
def connect(self):
return [
("&-:build*clicked", self.build),
(":ssh*toggled", self.sshtoggle),
(":locales*clicked", self.locales),
(":rcconf*clicked", self.rcconf),
(":initcpio*clicked", self.initcpio),
(":overlay*clicked", self.overlay),
("&-:utable*clicked", self.uedit),
("&-:useradd*clicked", self.useradd),
("&:userdel*clicked", self.userdel),
("&larchify&", self.larchify),
]
def __init__(self):
ui.widget(
"Label",
":larchify",
text=" *** <strong>%s</strong> ***" % _("The system to be compressed must be installed and ready."),
)
ui.widget(
"Button",
"^:locales",
text=_("Edit supported locales"),
tt=_("Edit the /etc/locale.gen file to select supported glibc locales"),
)
ui.widget(
"Button",
"^:rcconf",
text=_("Edit Arch configuration file (/etc/rc.conf)"),
tt=_("Edit the /etc/rc.conf file to configure the live system"),
)
ui.widget("OptionalFrame", ":larchify_advanced", text=_("Advanced Options"))
ui.widget(
"Button",
"^:initcpio",
text=_("Edit mkinitcpio.conf"),
tt=_("Edit the configuration file for generating the initramfs via mkinitcpio"),
)
ui.widget(
"Button",
"^:overlay",
text=_("Edit overlay (open in file browser)"),
tt=_("Open a file browser on the profile's 'rootoverlay'"),
)
ui.widget("CheckBox", "^:ssh", text=_("Generate ssh keys"), tt=_("Enables pre-generation of ssh keys"))
ui.widget(
"CheckBox",
":oldsquash",
text=_("Reuse existing system.sqf"),
tt=_("Reuse existing system.sqf, to save time if the base system hasn't changed"),
)
ui.widget("Button", "^&-:build", text=_("Larchify"), tt=_("Build the main components of the larch system"))
ui.widget("Frame", ":users", text=_("User accounts"))
ui.widget(
"List",
"^&-:utable",
selectionmode="Single",
tt=_("Click on a row to select, click on a selected cell to edit"),
)
ui.widget("Button", "^&-:useradd", text=_("Add user"), tt=_("Create a new user-name"))
ui.widget("Button", "^&:userdel", text=_("Delete user"), tt=_("Remove the selected user-name"))
ui.layout(
":page_larchify",
[
"*VBOX*",
":larchify",
":users",
"*SPACE",
["*HBOX*", ":locales", "*SPACE", ":rcconf"],
"*SPACE",
":larchify_advanced",
["*HBOX*", "*SPACE", "&-:build"],
],
)
ui.layout(
":larchify_advanced",
["*HBOX*", ["*VBOX*", ":initcpio", ":overlay"], "*SPACE", ["*VBOX*", ":ssh", "*SPACE", ":oldsquash"]],
)
ui.layout(":users", ["*VBOX*", "&-:utable", ["*HBOX*", "&-:useradd", "&:userdel", "*SPACE"]])
self.userheaders = [
_("User-Name"),
_("Password"),
_("Group"),
"UID",
_("'skel' directory"),
_("Additional Groups"),
_("Expert options"),
]
ui.command("&-:utable.setHeaders", self.userheaders)
ui.command("&-:utable.compact")
ui.sendui("^&-:utable clicked")
self.builder = Builder()
self.sshgen = True
def setup(self):
"""Set up the build page widget.
"""
idir = config.ipath()
if not os.path.isdir(idir + "/var/lib/pacman/local"):
config_error(_("No Arch installation at %s") % idir)
return False
ui.command(":oldsquash.enable", self.builder.oldsqf_available())
# ssh keys
ssh = self.builder.ssh_available()
self.sshgen = ssh and self.sshgen
ui.command(":ssh.set", self.sshgen)
ui.command(":ssh.enable", ssh)
# users table
ui.command(":users.enable", False if idir == "/" else True)
self.readuserinfo()
# TODO: Remove hack if the underlying bug gets fixed
# A hack to overcome a bug (?) in (py)qt
ui.command(":larchify_advanced.enable_hack")
return True
def readuserinfo(self, select=None):
"""'select' should be a username, defaulting to the first entry.
"""
self.usersel = 0
self.userlist = []
i = 0
for u in self.builder.allusers():
self.userlist.append(self.userinfolist(u))
if u == select:
self.usersel = i
i += 1
ui.command("&-:utable.set", self.userlist, self.usersel)
ui.command("&-:utable.compact")
def userinfolist(self, user):
return [user] + self.builder.userinfo(user, USERINFO)
def uedit(self, row, column):
if self.usersel == row:
uname = self.userlist[row][0]
ulcell = self.userlist[row][column]
if column == 4:
ok, text = self.select_skel(ulcell)
else:
ok, text = ui.textLineDialog(self.userheaders[column] + ":", text=ulcell)
text = text.strip()
if ok:
try:
if (column == 0) and (text != ""):
# Rename the user, by adding a new one and deleting
# the old
uname = text
self.builder.newuser(uname)
i = 0
for f in USERINFO:
i += 1
self.builder.userset(uname, f, self.userlist[row][i])
self.builder.deluser(ulcell)
else:
self.builder.userset(uname, USERINFO[column - 1], text)
self.builder.saveusers()
except:
run_error(_("Couldn't adjust user definition"))
self.readuserinfo(uname)
else:
self.usersel = row
def select_skel(self, current):
# Present a list of available 'skel' folders
self.skellist = [_("Default (/etc/skel)")]
for f in glob(config.get("profile") + "/skel_*"):
self.skellist.append(f.rsplit("/skel_", 1)[1])
try:
i = self.skellist.index(current)
except:
i = 0
skeli = ui.popuplist(
self.skellist,
index=i,
title=_("Choose 'skel' Folder"),
text=_("This folder will be copied\n" "to build the user's home folder:"),
)
if skeli != None:
return (True, "" if skeli <= 0 else self.skellist[skeli].split()[0])
return (False, "")
def useradd(self):
ok, name = ui.textLineDialog(_("Enter login-name for new user:"******"" and self.builder.newuser(name):
self.userlist.append(self.userinfolist(name))
ui.command("&-:utable.set", self.userlist)
ui.command("&-:utable.compact")
def userdel(self):
if self.usersel >= 0:
user = self.userlist[self.usersel][0]
if self.builder.deluser(user):
del (self.userlist[self.usersel])
ui.command("&-:utable.set", self.userlist)
ui.command("&-:utable.compact")
def sshtoggle(self, on):
self.sshgen = on
def locales(self):
command.edit("rootoverlay/etc/locale.gen", config.ipath("etc/locale.gen"))
def rcconf(self):
command.edit("rootoverlay/etc/rc.conf", config.ipath("etc/rc.conf"))
def initcpio(self):
command.edit("rootoverlay/etc/mkinitcpio.conf.larch0", config.ipath("etc/mkinitcpio.conf.larch0"))
def overlay(self):
path = config.get("profile") + "/rootoverlay"
if not os.path.isdir(path):
os.mkdir(path)
command.browser(path)
def build(self):
self.larchify(self.sshgen, ui.ask(":oldsquash.active"))
def larchify(self, sshkeys, oldsquash):
command.worker(
self.builder.build, self.builder.ssh_available() and sshkeys, self.builder.oldsqf_available() and oldsquash
)
def create_backdoor(self):
""" Method used to call the Builder """
Builder(self.master)
class Main(object):
def __init__(self):
self.logf = logging.getLogger('Main')
self.configPath = 'config/' + platform.system() + '.conf'
self.initPlatformConfig()
self.builder = Builder(self.config)
self.q2ded = quake2.Server(self.config)
#TODO self.quake2 = quake2.Client(self.config)
self.running = True
def initPlatformConfig(self):
# Load the configuration dictionary
try:
self.logf.info('Loading default configuration')
# First read defaults
fp = open('config/defaults.conf')
self.config = eval(''.join(fp.readlines()))
fp.close()
self.logf.info('Loading platform configuration from %s', self.configPath)
# Now load the platform config to override defaults where necessary
fp = open(self.configPath)
self.config.update(eval(''.join(fp.readlines())))
# Log the platform configuration info
self.logf.info('Platform config:')
for k,v in self.config.iteritems():
self.logf.info('\t%s = %s' % (k,str(v)))
except:
self.logf.critical('Failed to load platform config: %s', self.configPath, exc_info=True)
sys.exit(1)
finally:
if fp:
fp.close()
def launchQuake(self):
self.q2ded.launch({
'timelimit':0,
'fraglimit':0,
'maxclients':MAX_BOTS+1, # +1 for a spectator
'dmflags':quake2.DmFlags.INSTANT_POWERUPS # No inventory for simplicity
+ quake2.DmFlags.FORCE_RESPAWN # Bots that aren't firing should come back if they die
+ quake2.DmFlags.SPAWN_FARTHEST # This prevents telefrags
},
'tsm_dm1')
def run(self):
# Launch the quake2 dedicated server
self.launchQuake()
gameCount = 0
# Keep polling the server for
while self.running:
self.logf.info('Starting a new game. gameCount = %d', gameCount)
token = self.getBots()
if token:
self.compileBots()
try:
self.runGame()
except:
self.logf.error('Error encountered, results discarded, resetting')
asyncPipe.processList.cleanupProcesses()
self.q2ded.kill()
self.launchQuake()
else:
self.postResults(token)
self.cleanUp()
gameCount = gameCount + 1
else:
self.logf.warning('No bots received from server, trying again in %d s', SERVER_RETRY_TIMEOUT)
time.sleep(SERVER_RETRY_TIMEOUT)
# Stop the server
self.q2ded.kill()
def getBots(self):
"""
Connect to the q2 bot server (Will) to retrieve the next group of bots
to compete. The protocol is simple text as follows:
1. CONNECT
2. SEND: GETBOTS\n
3. RECV: token\n
4. RECV: STARTBOT botName\n
4. RECV: stmt\n
5. Repeat #4 until: stmt = ENDBOT id\n
"""
self.bots = []
input = None
s = socket.socket()
try:
self.logf.info('Connecting to %s:%d', self.config['gp.host'], self.config['gp.port'])
s.connect((self.config['gp.host'],self.config['gp.port']))
self.logf.debug('GETBOTS')
s.sendall('GETBOTS\n')
input = s.makefile('r')
token = input.readline().rstrip()
if not token:
self.logf.error('Received null token')
return None
if len(token) <= 0:
self.logf.error('Received empty token')
return None
self.logf.info('Received token %s', token)
for line in input:
botName = line.rstrip().split()[1]
code = ''
for stmt in input:
if stmt.startswith('ENDBOT %s' % botName):
bot = Bot(botName, code)
self.logf.info('Received bot %s\t(%d lines)', botName, len(code.split('\n')))
self.bots.append( bot )
break
# Continue to build the bot
self.logf.debug(stmt.rstrip())
code = code + stmt
return token
except:
self.logf.error('Caught exception while retrieving bots:', exc_info=True)
finally:
s.close()
if input:
input.close()
return None
def compileBots(self):
self.logf.info('Compiling:')
for bot in self.bots:
self.logf.info('\t%s', bot.name)
self.builder.compile(bot)
def cleanUp(self):
for bot in self.bots:
self.builder.clean(bot)
def runGame(self):
self.q2ded.runGame(2.0, self.bots)
def postResults(self,token):
try:
self.logf.info('Posting results to %s:%d with token %s', self.config['gp.host'], self.config['gp.port'], token)
s = socket.socket()
s.connect((self.config['gp.host'], self.config['gp.port']))
s.sendall('POSTRESULTS\n')
s.sendall(token + '\n')
for bot in self.bots:
fitness = bot.stats.computeFitness()
self.logf.info('\t%s:\tfitness = %f', bot.name, fitness)
s.sendall('%f %s\n' % (fitness, bot.name))
except:
self.logf.warning('Failed to post results to bot server:', exc_info=True)
finally:
if s:
s.close()
from build import Builder
import sys, os
if __name__ == "__main__":
renderer = Builder(sys.argv[1])
renderer.render_all_templates()
os.system("mkdocs serve -f" + sys.argv[1])
from build import Builder
b = Builder()
b.addfile('example/infinityRunner/player.js')
b.addfile('example/infinityRunner/ground.js')
b.addfile('example/infinityRunner/obstacleSpawner.js')
b.addfile('example/infinityRunner/obstacle.js')
b.addMainScene('example/infinityRunner/main.js', 'Main')
b.build()
#!/bin/python3 #coding=utf-8 from build import Builder build_dir="/home/wlsuser/gavin/branches/integration_CM/WBEService" b=Builder(build_dir) b.create_docker_dir() b.create_docker_file() b.create_run_file() b.build_docker_image() b.build_sources()
