def prepare(self):
try:
os.mkdir("/root/build")
except OSError:
pass
os.chdir("/root/build")
os.system("rm -Rf spl")
gitinfo = self.job.attributes['zfs-git-repo']
repodir = "%s/repositories/%s" % (Utility.get_persistent_dir(),
gitinfo['spl-repository'])
if os.system("git clone -b %s %s spl" %
(gitinfo['spl-branch'], repodir)) != 0:
return TaskResult.FAILED
os.chdir("spl")
if os.system(
"git reset --hard `git rev-list --max-count=1 --before=%s HEAD`"
% (gitinfo['timestamp'])) != 0:
return TaskResult.FAILED
return TaskResult.PASSED
def set_configured_jobresult(self, in_chroot): idfile = Utility.get_result_dir() + '/result-id' if not in_chroot: idfile = '/mnt/' + idfile fp = open(idfile, 'w') fp.write(self.hash) fp.close
def get_values(self):
config = Utility.get_zfsci_config()
kernelbasepath = config['persistent_path'] + '/kernels'
if not os.path.isdir(kernelbasepath):
try:
os.makedirs(kernelbasepath)
except OSError:
pass
sourcepath = Utility.get_source_dir()
os.system("cp -a %s/misc/modules %s" % (sourcepath, kernelbasepath))
os.chdir(kernelbasepath)
kernels = {
'2.6.26': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.26/',
'2.6.27': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.27/',
'2.6.28': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.28/',
'2.6.29': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.29/',
'2.6.30': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.30/',
'2.6.31': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.31/',
'2.6.32': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.32/',
'2.6.33': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.33/',
'2.6.34': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.34-lucid/',
'2.6.35': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.35-maverick/',
'2.6.36': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.36-maverick/',
'2.6.37': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.37-natty/',
'2.6.38': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.38-natty/',
'2.6.39': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.39-rc5-oneiric/'
}
for kernel in kernels.iterkeys():
if os.path.islink(kernel):
continue
os.system("wget -r -np -c '%s'" % (kernels[kernel]))
os.symlink(kernels[kernel][7:], kernel)
return kernels.keys()
def run_stage(self, stage):
self.jobresult.attributes['stage_%s_start' % (stage)] = datetime.now()
for taskfile in glob.glob('%s/*.py' % (Utility.get_tasks_dir())):
execfile(taskfile, globals())
for task in JobDispatcher.tasks:
if task.stage != stage:
continue
self._run_task_once(task)
self.jobresult.attributes['stage_%s_end' % (stage)] = datetime.now()
def get_configured_jobresult(session, in_chroot): idfile = Utility.get_result_dir() + '/result-id' if not in_chroot: idfile = '/mnt/' + idfile if not os.path.isfile(idfile): return None fp = open(idfile, 'r') result_hash = fp.read().strip() fp.close() return JobResult.get_by_hash(session, result_hash)
def get_values(self):
config = Utility.get_zfsci_config()
distsbasepath = config['persistent_path'] + '/dists'
if not os.path.isdir(distsbasepath):
try:
os.makedirs(distsbasepath)
except OSError:
pass
distsscriptdir = Utility.get_source_dir() + '/dists'
distributions = [
'debian',
#'opensuse', # broken
#'centos', # mostly works
#'scientific' # horribly broken
]
for distribution in distributions:
if not os.path.isfile("%s/%s.tar.gz" % (distsbasepath, distribution)):
os.system("%s/%s.sh %s/%s.tar.gz" % (distsscriptdir, distribution, distsbasepath, distribution))
return distributions
def _run_task_once(self, task): if task.result != None and task.result.status != TaskResult.PENDING: return task.job = self.jobresult.job task.result = self.jobresult.get_task(str(task.__class__), True) task.result.stage = task.stage task.dependencies_resolved = True for dependency in task.dependencies: depresolved = False for deptask in JobDispatcher.tasks: if not dependency in deptask.provides: continue self._run_task_once(deptask) if deptask.result.status == TaskResult.PASSED: depresolved = True if not depresolved: task.dependencies_resolved = False print "Running task '%s' (type %s)" % \ (task.description, task.__class__) cwd = os.getcwd() Utility.rearm_watchdog(1800) task._run() Utility.rearm_watchdog(0) os.chdir(cwd) print "Task result: %s" % (task.result.status) JobSession.object_session(self.jobresult).commit()
def mount_partitions():
if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
try:
os.makedirs("/mnt")
except OSError:
pass
if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
raise Error("Could not mount node filesystem.")
if os.system("mountpoint -q /mnt/dev") != 0:
try:
os.makedirs("/mnt/dev")
except OSError:
pass
if os.system("mount --bind /dev /mnt/dev") != 0:
raise Error("Could not mount /mnt/dev")
if os.system("mountpoint -q /mnt/sys") != 0:
try:
os.makedirs("/mnt/sys")
except OSError:
pass
if os.system("mount --bind /sys /mnt/sys") != 0:
raise Error("Could not mount /mnt/sys")
if os.system("mountpoint -q /mnt/proc") != 0:
try:
os.makedirs("/mnt/proc")
except OSError:
pass
if os.system("mount -t proc none /mnt/proc") != 0:
raise Error("Could not mount /mnt/proc")
persistent_dir = Utility.get_persistent_dir()
if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
try:
os.makedirs("/mnt/%s" % (persistent_dir))
except OSError:
pass
if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
raise Error("Could not mount /mnt/%s" % (persistent_dir))
def mount_partitions():
if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
try:
os.makedirs("/mnt")
except OSError:
pass
if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
raise Error("Could not mount node filesystem.")
if os.system("mountpoint -q /mnt/dev") != 0:
try:
os.makedirs("/mnt/dev")
except OSError:
pass
if os.system("mount --bind /dev /mnt/dev") != 0:
raise Error("Could not mount /mnt/dev")
if os.system("mountpoint -q /mnt/sys") != 0:
try:
os.makedirs("/mnt/sys")
except OSError:
pass
if os.system("mount --bind /sys /mnt/sys") != 0:
raise Error("Could not mount /mnt/sys")
if os.system("mountpoint -q /mnt/proc") != 0:
try:
os.makedirs("/mnt/proc")
except OSError:
pass
if os.system("mount -t proc none /mnt/proc") != 0:
raise Error("Could not mount /mnt/proc")
persistent_dir = Utility.get_persistent_dir()
if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
try:
os.makedirs("/mnt/%s" % (persistent_dir))
except OSError:
pass
if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
raise Error("Could not mount /mnt/%s" % (persistent_dir))
def prepare(self):
try:
os.mkdir("/root/build")
except OSError:
pass
os.chdir("/root/build")
os.system("rm -Rf spl")
gitinfo = self.job.attributes['zfs-git-repo']
repodir = "%s/repositories/%s" % (Utility.get_persistent_dir(), gitinfo['spl-repository'])
if os.system("git clone -b %s %s spl" % (gitinfo['spl-branch'], repodir)) != 0:
return TaskResult.FAILED
os.chdir("spl")
if os.system("git reset --hard `git rev-list --max-count=1 --before=%s HEAD`" % (gitinfo['timestamp'])) != 0:
return TaskResult.FAILED
return TaskResult.PASSED
def _run(self): assert self.result.status == TaskResult.PENDING if not self.should_run(): self.result.status = TaskResult.SKIPPED return if not self.dependencies_resolved: self.result.status = TaskResult.DEPENDENCY_ERROR return clshash = hashlib.md5(str(self.__class__)).hexdigest() outputfile = Utility.get_result_dir() + '/output_' + clshash outputfp = open(outputfile, "w") stdoutfd = os.dup(1) stderrfd = os.dup(2) os.dup2(outputfp.fileno(), 1) os.dup2(outputfp.fileno(), 2) self.result.status = TaskResult.RUNNING JobSession.object_session(self.result).commit() try: status = self.prepare() if status == None or status == TaskResult.PASSED: self.result.run_start = datetime.now() status = self.run() self.result.run_end = datetime.now() if status == None: status = TaskResult.FAILED except Exception, exc: status = TaskResult.FAILED self.result.attributes['exception'] = str(exc) self.result.attributes['stacktrace'] = traceback.format_exc() print exc
def process_results(session): for resultdir in glob.glob(Utility.get_persistent_dir() + '/results/*'): try: resultfile = resultdir + '/job.json' fp = open(resultfile, 'r') resultinfo = json.load(fp) fp.close() statinfo = os.lstat(resultfile) timestamp = datetime.fromtimestamp(statinfo.st_mtime) except Exception, exc: print exc continue job = Job.get_by_hash(session, resultinfo['job_id']) if job == None: job = Job(resultinfo['job_id']) job.attributes = resultinfo['input'] session.add(job) if not 'output' in resultinfo: continue if not 'result_id' in resultinfo: resultinfo['result_id'] = hashlib.md5(os.path.basename(resultdir)).hexdigest() result = JobResult.get_by_hash(session, resultinfo['result_id']) if result == None: jobresult = JobResult(job, resultinfo['result_id']) for (task, taskinfo) in resultinfo['output'].iteritems(): taskresult = TaskResult(jobresult, task) taskresult.attributes = taskinfo session.add(taskresult) session.add(jobresult)
def unmount_partitions():
persistent_dir = Utility.get_persistent_dir()
os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
class PartitionBuilder(object):
_config = Utility.get_zfsci_config()
@staticmethod
def get_install_device():
return PartitionBuilder._config['install_device']
@staticmethod
def get_rootpart():
return PartitionBuilder.get_install_device() + '1'
@staticmethod
def get_swappart():
return PartitionBuilder.get_install_device() + '2'
@staticmethod
def get_testpart():
return PartitionBuilder.get_install_device() + '3'
@staticmethod
def setup_partitions():
install_device = PartitionBuilder.get_install_device()
if os.system("dd if=/dev/zero of=%s bs=512 count=1" % (install_device)) != 0:
raise Error("Could not erase old partition table.")
if os.system("parted --script -- %s mktable msdos" % (install_device)) != 0:
raise Error("Could not create new partition table.")
rootpart = PartitionBuilder.get_rootpart()
if os.system("parted --script -- %s mkpart primary ext2 1M 10G" % (install_device)) != 0:
raise Error("Could not create new / partition.")
swappart = PartitionBuilder.get_swappart()
if os.system("parted --script -- %s mkpart primary linux-swap 10G 14G" % (install_device)) != 0:
raise Error("Could not create new swap partition.")
testpart = PartitionBuilder.get_testpart()
if os.system("parted --script -- %s mkpart primary sun-ufs 14G -1s" % (install_device)) != 0:
raise Error("Could not create new test partition.")
if os.system("mke2fs -j -m 0 -L / -I 128 %s" % (rootpart)) != 0:
raise Error("Create not create new / filesystem.")
if os.system("mkswap %s" % (swappart)) != 0:
raise Error("Could not create new swap filesystem.")
@staticmethod
def mount_partitions():
if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
try:
os.makedirs("/mnt")
except OSError:
pass
if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
raise Error("Could not mount node filesystem.")
if os.system("mountpoint -q /mnt/dev") != 0:
try:
os.makedirs("/mnt/dev")
except OSError:
pass
if os.system("mount --bind /dev /mnt/dev") != 0:
raise Error("Could not mount /mnt/dev")
if os.system("mountpoint -q /mnt/sys") != 0:
try:
os.makedirs("/mnt/sys")
except OSError:
pass
if os.system("mount --bind /sys /mnt/sys") != 0:
raise Error("Could not mount /mnt/sys")
if os.system("mountpoint -q /mnt/proc") != 0:
try:
os.makedirs("/mnt/proc")
except OSError:
pass
if os.system("mount -t proc none /mnt/proc") != 0:
raise Error("Could not mount /mnt/proc")
persistent_dir = Utility.get_persistent_dir()
if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
try:
os.makedirs("/mnt/%s" % (persistent_dir))
except OSError:
pass
if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
raise Error("Could not mount /mnt/%s" % (persistent_dir))
@staticmethod
def unmount_partitions():
persistent_dir = Utility.get_persistent_dir()
os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
def unmount_partitions():
persistent_dir = Utility.get_persistent_dir()
os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
def run(self):
for file in glob.glob("/var/crash/[0-9]*/*"):
shutil.copy(file, Utility.get_result_dir())
return TaskResult.PASSED
def run(self):
for file in glob.glob("/var/crash/[0-9]*/*"):
shutil.copy(file, Utility.get_result_dir())
return TaskResult.PASSED
def get_job_session(debug=False): config = Utility.get_zfsci_config() engine = create_engine(config['job_dsn'], echo=debug) ModelBase.metadata.create_all(engine) return JobSession(bind=engine)
def get_values(self):
repositories = [
{
"name": "behlendorf",
"spl": "git://github.com/behlendorf/spl.git",
"zfs": "git://github.com/behlendorf/zfs.git",
"zfs-ignore-branches": ["gh-pages"],
},
{
"name": "gunnarbeutner",
"spl": "git://github.com/gunnarbeutner/pkg-spl.git",
"zfs": "git://github.com/gunnarbeutner/pkg-zfs.git",
},
]
config = Utility.get_zfsci_config()
repobasepath = config["persistent_path"] + "/repositories"
if not os.path.isdir(repobasepath):
try:
os.makedirs(repobasepath)
except OSError:
pass
os.chdir(repobasepath)
values = [None]
for repository in repositories:
for subrepo in ["spl", "zfs"]:
repodir = "%s-%s" % (repository["name"], subrepo)
repopath = repobasepath + "/" + repodir
if not os.path.isdir(repopath):
os.chdir(repobasepath)
os.system("git clone --mirror %s %s" % (repository[subrepo], repopath))
os.chdir(repopath)
os.system("git fetch")
args = ["git", "for-each-ref", "--format=%(refname:short)", "refs/heads/"]
output = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
branches = output.strip().split("\n")
for branch in branches:
if (
subrepo + "-ignore-branches" in repository
and branch in repository[subrepo + "-ignore-branches"]
):
continue
args = ["git", "rev-list", "--max-count=5", "--timestamp", branch]
output = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
commits = output.strip().split("\n")
for commitinfo in commits:
(timestamp, commit) = commitinfo.split(" ", 1)
value = {
"spl-repository": "%s-spl" % (repository["name"]),
"zfs-repository": "%s-zfs" % (repository["name"]),
"spl-branch": "master",
"zfs-branch": "master",
"timestamp": timestamp,
}
value[subrepo + "-branch"] = branch
values.append(value)
return values
def get_values(self):
repositories = [{
'name': 'behlendorf',
'spl': 'git://github.com/behlendorf/spl.git',
'zfs': 'git://github.com/behlendorf/zfs.git',
'zfs-ignore-branches': ['gh-pages']
}, {
'name': 'gunnarbeutner',
'spl': 'git://github.com/gunnarbeutner/pkg-spl.git',
'zfs': 'git://github.com/gunnarbeutner/pkg-zfs.git'
}]
config = Utility.get_zfsci_config()
repobasepath = config['persistent_path'] + '/repositories'
if not os.path.isdir(repobasepath):
try:
os.makedirs(repobasepath)
except OSError:
pass
os.chdir(repobasepath)
values = [None]
for repository in repositories:
for subrepo in ['spl', 'zfs']:
repodir = '%s-%s' % (repository['name'], subrepo)
repopath = repobasepath + '/' + repodir
if not os.path.isdir(repopath):
os.chdir(repobasepath)
os.system('git clone --mirror %s %s' %
(repository[subrepo], repopath))
os.chdir(repopath)
os.system('git fetch')
args = [
'git', 'for-each-ref', '--format=%(refname:short)',
'refs/heads/'
]
output = subprocess.Popen(
args, stdout=subprocess.PIPE).communicate()[0]
branches = output.strip().split('\n')
for branch in branches:
if subrepo + '-ignore-branches' in repository and \
branch in repository[subrepo + '-ignore-branches']:
continue
args = [
'git', 'rev-list', '--max-count=5', '--timestamp',
branch
]
output = subprocess.Popen(
args, stdout=subprocess.PIPE).communicate()[0]
commits = output.strip().split('\n')
for commitinfo in commits:
(timestamp, commit) = commitinfo.split(' ', 1)
value = {
'spl-repository': '%s-spl' % (repository['name']),
'zfs-repository': '%s-zfs' % (repository['name']),
'spl-branch': 'master',
'zfs-branch': 'master',
'timestamp': timestamp
}
value[subrepo + '-branch'] = branch
values.append(value)
return values