Esempio n. 1
0
    def prepare(self):
        try:
            os.mkdir("/root/build")
        except OSError:
            pass

        os.chdir("/root/build")
        os.system("rm -Rf spl")

        gitinfo = self.job.attributes['zfs-git-repo']

        repodir = "%s/repositories/%s" % (Utility.get_persistent_dir(),
                                          gitinfo['spl-repository'])
        if os.system("git clone -b %s %s spl" %
                     (gitinfo['spl-branch'], repodir)) != 0:
            return TaskResult.FAILED

        os.chdir("spl")

        if os.system(
                "git reset --hard `git rev-list --max-count=1 --before=%s HEAD`"
                % (gitinfo['timestamp'])) != 0:
            return TaskResult.FAILED

        return TaskResult.PASSED
Esempio n. 2
0
	def set_configured_jobresult(self, in_chroot):
		idfile = Utility.get_result_dir()  + '/result-id'

		if not in_chroot:
			idfile = '/mnt/' + idfile

		fp = open(idfile, 'w')
		fp.write(self.hash)
		fp.close
Esempio n. 3
0
	def get_values(self):
		config = Utility.get_zfsci_config()
		kernelbasepath = config['persistent_path'] + '/kernels'

		if not os.path.isdir(kernelbasepath):
			try:
				os.makedirs(kernelbasepath)
			except OSError:
				pass

		sourcepath = Utility.get_source_dir()
		os.system("cp -a %s/misc/modules %s" % (sourcepath, kernelbasepath))

		os.chdir(kernelbasepath)

		kernels = {
			'2.6.26': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.26/',
			'2.6.27': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.27/',
			'2.6.28': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.28/',
			'2.6.29': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.29/',
			'2.6.30': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.30/',
			'2.6.31': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.31/',
			'2.6.32': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.32/',
			'2.6.33': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.33/',
			'2.6.34': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.34-lucid/',
			'2.6.35': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.35-maverick/',
			'2.6.36': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.36-maverick/',
			'2.6.37': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.37-natty/',
			'2.6.38': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.38-natty/',
			'2.6.39': 'http://kernel.ubuntu.com/~kernel-ppa/mainline/v2.6.39-rc5-oneiric/'
		}

		for kernel in kernels.iterkeys():
			if os.path.islink(kernel):
				continue

			os.system("wget -r -np -c '%s'" % (kernels[kernel]))
			os.symlink(kernels[kernel][7:], kernel)

		return kernels.keys()
Esempio n. 4
0
	def run_stage(self, stage):
		self.jobresult.attributes['stage_%s_start' % (stage)] = datetime.now()

		for taskfile in glob.glob('%s/*.py' % (Utility.get_tasks_dir())):
			execfile(taskfile, globals())

		for task in JobDispatcher.tasks:
			if task.stage != stage:
				continue

			self._run_task_once(task)

		self.jobresult.attributes['stage_%s_end' % (stage)] = datetime.now()
Esempio n. 5
0
	def get_configured_jobresult(session, in_chroot):
		idfile = Utility.get_result_dir()  + '/result-id'

		if not in_chroot:
			idfile = '/mnt/' + idfile

		if not os.path.isfile(idfile):
			return None

		fp = open(idfile, 'r')
		result_hash = fp.read().strip()
		fp.close()

		return JobResult.get_by_hash(session, result_hash)
Esempio n. 6
0
	def get_values(self):
		config = Utility.get_zfsci_config()
		distsbasepath = config['persistent_path'] + '/dists'

		if not os.path.isdir(distsbasepath):
			try:
				os.makedirs(distsbasepath)
			except OSError:
				pass

		distsscriptdir = Utility.get_source_dir() + '/dists'

		distributions = [
			'debian',
			#'opensuse', # broken
			#'centos', # mostly works
			#'scientific' # horribly broken
		]

		for distribution in distributions:
			if not os.path.isfile("%s/%s.tar.gz" % (distsbasepath, distribution)):
				os.system("%s/%s.sh %s/%s.tar.gz" % (distsscriptdir, distribution, distsbasepath, distribution))

		return distributions
Esempio n. 7
0
	def _run_task_once(self, task):
		if task.result != None and task.result.status != TaskResult.PENDING:
			return

		task.job = self.jobresult.job
		task.result = self.jobresult.get_task(str(task.__class__), True)
		task.result.stage = task.stage
		task.dependencies_resolved = True

		for dependency in task.dependencies:
			depresolved = False

			for deptask in JobDispatcher.tasks:
				if not dependency in deptask.provides:
					continue

				self._run_task_once(deptask)

				if deptask.result.status == TaskResult.PASSED:
					depresolved = True

			if not depresolved:
				task.dependencies_resolved = False

		print "Running task '%s' (type %s)" % \
			(task.description, task.__class__)

		cwd = os.getcwd()
		Utility.rearm_watchdog(1800)
		task._run()
		Utility.rearm_watchdog(0)
		os.chdir(cwd)

		print "Task result: %s" % (task.result.status)

		JobSession.object_session(self.jobresult).commit()
Esempio n. 8
0
	def mount_partitions():
		if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
			try:
				os.makedirs("/mnt")
			except OSError:
				pass

			if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
				raise Error("Could not mount node filesystem.")

		if os.system("mountpoint -q /mnt/dev") != 0:
			try:
				os.makedirs("/mnt/dev")
			except OSError:
				pass

			if os.system("mount --bind /dev /mnt/dev") != 0:
				raise Error("Could not mount /mnt/dev")

		if os.system("mountpoint -q /mnt/sys") != 0:
			try:
				os.makedirs("/mnt/sys")
			except OSError:
				pass

			if os.system("mount --bind /sys /mnt/sys") != 0:
				raise Error("Could not mount /mnt/sys")

		if os.system("mountpoint -q /mnt/proc") != 0:
			try:
				os.makedirs("/mnt/proc")
			except OSError:
				pass

			if os.system("mount -t proc none /mnt/proc") != 0:
				raise Error("Could not mount /mnt/proc")

		persistent_dir = Utility.get_persistent_dir()
		if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
			try:
				os.makedirs("/mnt/%s" % (persistent_dir))
			except OSError:
				pass

			if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
				raise Error("Could not mount /mnt/%s" % (persistent_dir))
Esempio n. 9
0
    def mount_partitions():
        if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
            try:
                os.makedirs("/mnt")
            except OSError:
                pass

            if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
                raise Error("Could not mount node filesystem.")

        if os.system("mountpoint -q /mnt/dev") != 0:
            try:
                os.makedirs("/mnt/dev")
            except OSError:
                pass

            if os.system("mount --bind /dev /mnt/dev") != 0:
                raise Error("Could not mount /mnt/dev")

        if os.system("mountpoint -q /mnt/sys") != 0:
            try:
                os.makedirs("/mnt/sys")
            except OSError:
                pass

            if os.system("mount --bind /sys /mnt/sys") != 0:
                raise Error("Could not mount /mnt/sys")

        if os.system("mountpoint -q /mnt/proc") != 0:
            try:
                os.makedirs("/mnt/proc")
            except OSError:
                pass

            if os.system("mount -t proc none /mnt/proc") != 0:
                raise Error("Could not mount /mnt/proc")

        persistent_dir = Utility.get_persistent_dir()
        if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
            try:
                os.makedirs("/mnt/%s" % (persistent_dir))
            except OSError:
                pass

            if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
                raise Error("Could not mount /mnt/%s" % (persistent_dir))
Esempio n. 10
0
	def prepare(self):
		try:
			os.mkdir("/root/build")
		except OSError:
			pass

		os.chdir("/root/build")
		os.system("rm -Rf spl")

		gitinfo = self.job.attributes['zfs-git-repo']

		repodir = "%s/repositories/%s" % (Utility.get_persistent_dir(), gitinfo['spl-repository'])
		if os.system("git clone -b %s %s spl" % (gitinfo['spl-branch'], repodir)) != 0:
			return TaskResult.FAILED

		os.chdir("spl")

		if os.system("git reset --hard `git rev-list --max-count=1 --before=%s HEAD`" % (gitinfo['timestamp'])) != 0:
			return TaskResult.FAILED

		return TaskResult.PASSED
Esempio n. 11
0
	def _run(self):
		assert self.result.status == TaskResult.PENDING

		if not self.should_run():
			self.result.status = TaskResult.SKIPPED
			return

		if not self.dependencies_resolved:
			self.result.status = TaskResult.DEPENDENCY_ERROR
			return

		clshash = hashlib.md5(str(self.__class__)).hexdigest()
		outputfile = Utility.get_result_dir() + '/output_' + clshash
		outputfp = open(outputfile, "w")

		stdoutfd = os.dup(1)
		stderrfd = os.dup(2)

		os.dup2(outputfp.fileno(), 1)
		os.dup2(outputfp.fileno(), 2)

		self.result.status = TaskResult.RUNNING
		JobSession.object_session(self.result).commit()

		try:
			status = self.prepare()

			if status == None or status == TaskResult.PASSED:
				self.result.run_start = datetime.now()
				status = self.run()
				self.result.run_end = datetime.now()
			if status == None:
				status = TaskResult.FAILED
		except Exception, exc:
			status = TaskResult.FAILED

			self.result.attributes['exception'] = str(exc)
			self.result.attributes['stacktrace'] = traceback.format_exc()

			print exc
Esempio n. 12
0
def process_results(session):
	for resultdir in glob.glob(Utility.get_persistent_dir() + '/results/*'):
		try:
			resultfile = resultdir + '/job.json'

			fp = open(resultfile, 'r')
			resultinfo = json.load(fp)
			fp.close()

			statinfo = os.lstat(resultfile)
			timestamp = datetime.fromtimestamp(statinfo.st_mtime)
		except Exception, exc:
			print exc
			continue

		job = Job.get_by_hash(session, resultinfo['job_id'])

		if job == None:
			job = Job(resultinfo['job_id'])
			job.attributes = resultinfo['input']
			session.add(job)

		if not 'output' in resultinfo:
			continue

		if not 'result_id' in resultinfo:
			resultinfo['result_id'] = hashlib.md5(os.path.basename(resultdir)).hexdigest()

		result = JobResult.get_by_hash(session, resultinfo['result_id'])

		if result == None:
			jobresult = JobResult(job, resultinfo['result_id'])

			for (task, taskinfo) in resultinfo['output'].iteritems():
				taskresult = TaskResult(jobresult, task)
				taskresult.attributes =  taskinfo
				session.add(taskresult)

			session.add(jobresult)
Esempio n. 13
0
 def unmount_partitions():
     persistent_dir = Utility.get_persistent_dir()
     os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
Esempio n. 14
0
class PartitionBuilder(object):
	_config = Utility.get_zfsci_config()

	@staticmethod
	def get_install_device():
		return PartitionBuilder._config['install_device']

	@staticmethod
	def get_rootpart():
		return PartitionBuilder.get_install_device() + '1'

	@staticmethod
	def get_swappart():
		return PartitionBuilder.get_install_device() + '2'

	@staticmethod
	def get_testpart():
		return PartitionBuilder.get_install_device() + '3'

	@staticmethod
	def setup_partitions():
		install_device = PartitionBuilder.get_install_device()

		if os.system("dd if=/dev/zero of=%s bs=512 count=1" % (install_device)) != 0:
			raise Error("Could not erase old partition table.")

		if os.system("parted --script -- %s mktable msdos" % (install_device)) != 0:
			raise Error("Could not create new partition table.")

		rootpart = PartitionBuilder.get_rootpart()
		if os.system("parted --script -- %s mkpart primary ext2 1M 10G" % (install_device)) != 0:
			raise Error("Could not create new / partition.")

		swappart = PartitionBuilder.get_swappart()
		if os.system("parted --script -- %s mkpart primary linux-swap 10G 14G" % (install_device)) != 0:
			raise Error("Could not create new swap partition.")

		testpart = PartitionBuilder.get_testpart()
		if os.system("parted --script -- %s mkpart primary sun-ufs 14G -1s" % (install_device)) != 0:
			raise Error("Could not create new test partition.")

		if os.system("mke2fs -j -m 0 -L / -I 128 %s" % (rootpart)) != 0:
			raise Error("Create not create new / filesystem.")

		if os.system("mkswap %s" % (swappart)) != 0:
			raise Error("Could not create new swap filesystem.")

	@staticmethod
	def mount_partitions():
		if os.system("mountpoint -q /mnt") != 0 or os.system("mount -o remount,rw /mnt") != 0:
			try:
				os.makedirs("/mnt")
			except OSError:
				pass

			if os.system("mount %s /mnt" % (PartitionBuilder.get_rootpart())) != 0:
				raise Error("Could not mount node filesystem.")

		if os.system("mountpoint -q /mnt/dev") != 0:
			try:
				os.makedirs("/mnt/dev")
			except OSError:
				pass

			if os.system("mount --bind /dev /mnt/dev") != 0:
				raise Error("Could not mount /mnt/dev")

		if os.system("mountpoint -q /mnt/sys") != 0:
			try:
				os.makedirs("/mnt/sys")
			except OSError:
				pass

			if os.system("mount --bind /sys /mnt/sys") != 0:
				raise Error("Could not mount /mnt/sys")

		if os.system("mountpoint -q /mnt/proc") != 0:
			try:
				os.makedirs("/mnt/proc")
			except OSError:
				pass

			if os.system("mount -t proc none /mnt/proc") != 0:
				raise Error("Could not mount /mnt/proc")

		persistent_dir = Utility.get_persistent_dir()
		if os.system("mountpoint -q /mnt/%s" % (persistent_dir)) != 0:
			try:
				os.makedirs("/mnt/%s" % (persistent_dir))
			except OSError:
				pass

			if os.system("mount --bind %s /mnt/%s" % (persistent_dir, persistent_dir)) != 0:
				raise Error("Could not mount /mnt/%s" % (persistent_dir))

	@staticmethod
	def unmount_partitions():
		persistent_dir = Utility.get_persistent_dir()
		os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
Esempio n. 15
0
	def unmount_partitions():
		persistent_dir = Utility.get_persistent_dir()
		os.system("umount /mnt/%s /mnt/dev /mnt/sys /mnt/proc /mnt" % (persistent_dir))
Esempio n. 16
0
    def run(self):
        for file in glob.glob("/var/crash/[0-9]*/*"):
            shutil.copy(file, Utility.get_result_dir())

        return TaskResult.PASSED
Esempio n. 17
0
	def run(self):
		for file in glob.glob("/var/crash/[0-9]*/*"):
			shutil.copy(file, Utility.get_result_dir())

		return TaskResult.PASSED
Esempio n. 18
0
def get_job_session(debug=False):
	config = Utility.get_zfsci_config()
	engine = create_engine(config['job_dsn'], echo=debug)
	ModelBase.metadata.create_all(engine)

	return JobSession(bind=engine)
Esempio n. 19
0
    def get_values(self):
        repositories = [
            {
                "name": "behlendorf",
                "spl": "git://github.com/behlendorf/spl.git",
                "zfs": "git://github.com/behlendorf/zfs.git",
                "zfs-ignore-branches": ["gh-pages"],
            },
            {
                "name": "gunnarbeutner",
                "spl": "git://github.com/gunnarbeutner/pkg-spl.git",
                "zfs": "git://github.com/gunnarbeutner/pkg-zfs.git",
            },
        ]

        config = Utility.get_zfsci_config()
        repobasepath = config["persistent_path"] + "/repositories"

        if not os.path.isdir(repobasepath):
            try:
                os.makedirs(repobasepath)
            except OSError:
                pass

        os.chdir(repobasepath)

        values = [None]
        for repository in repositories:
            for subrepo in ["spl", "zfs"]:
                repodir = "%s-%s" % (repository["name"], subrepo)
                repopath = repobasepath + "/" + repodir

                if not os.path.isdir(repopath):
                    os.chdir(repobasepath)
                    os.system("git clone --mirror %s %s" % (repository[subrepo], repopath))

                os.chdir(repopath)
                os.system("git fetch")

                args = ["git", "for-each-ref", "--format=%(refname:short)", "refs/heads/"]
                output = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
                branches = output.strip().split("\n")

                for branch in branches:
                    if (
                        subrepo + "-ignore-branches" in repository
                        and branch in repository[subrepo + "-ignore-branches"]
                    ):
                        continue

                    args = ["git", "rev-list", "--max-count=5", "--timestamp", branch]
                    output = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]

                    commits = output.strip().split("\n")

                    for commitinfo in commits:
                        (timestamp, commit) = commitinfo.split(" ", 1)

                        value = {
                            "spl-repository": "%s-spl" % (repository["name"]),
                            "zfs-repository": "%s-zfs" % (repository["name"]),
                            "spl-branch": "master",
                            "zfs-branch": "master",
                            "timestamp": timestamp,
                        }

                        value[subrepo + "-branch"] = branch

                        values.append(value)

        return values
Esempio n. 20
0
    def get_values(self):
        repositories = [{
            'name': 'behlendorf',
            'spl': 'git://github.com/behlendorf/spl.git',
            'zfs': 'git://github.com/behlendorf/zfs.git',
            'zfs-ignore-branches': ['gh-pages']
        }, {
            'name': 'gunnarbeutner',
            'spl': 'git://github.com/gunnarbeutner/pkg-spl.git',
            'zfs': 'git://github.com/gunnarbeutner/pkg-zfs.git'
        }]

        config = Utility.get_zfsci_config()
        repobasepath = config['persistent_path'] + '/repositories'

        if not os.path.isdir(repobasepath):
            try:
                os.makedirs(repobasepath)
            except OSError:
                pass

        os.chdir(repobasepath)

        values = [None]
        for repository in repositories:
            for subrepo in ['spl', 'zfs']:
                repodir = '%s-%s' % (repository['name'], subrepo)
                repopath = repobasepath + '/' + repodir

                if not os.path.isdir(repopath):
                    os.chdir(repobasepath)
                    os.system('git clone --mirror %s %s' %
                              (repository[subrepo], repopath))

                os.chdir(repopath)
                os.system('git fetch')

                args = [
                    'git', 'for-each-ref', '--format=%(refname:short)',
                    'refs/heads/'
                ]
                output = subprocess.Popen(
                    args, stdout=subprocess.PIPE).communicate()[0]
                branches = output.strip().split('\n')

                for branch in branches:
                    if subrepo + '-ignore-branches' in repository and \
                      branch in repository[subrepo + '-ignore-branches']:
                        continue

                    args = [
                        'git', 'rev-list', '--max-count=5', '--timestamp',
                        branch
                    ]
                    output = subprocess.Popen(
                        args, stdout=subprocess.PIPE).communicate()[0]

                    commits = output.strip().split('\n')

                    for commitinfo in commits:
                        (timestamp, commit) = commitinfo.split(' ', 1)

                        value = {
                            'spl-repository': '%s-spl' % (repository['name']),
                            'zfs-repository': '%s-zfs' % (repository['name']),
                            'spl-branch': 'master',
                            'zfs-branch': 'master',
                            'timestamp': timestamp
                        }

                        value[subrepo + '-branch'] = branch

                        values.append(value)

        return values