def test_traverse_buld_dir_with_oserror(self): try: errors = {} temp_dir = tempfile.mkdtemp() build_dir = os.path.join(temp_dir, "build_dir") os.mkdir(build_dir) build_file = os.path.join(build_dir, "build.json") with io.open(build_file, mode="w") as write_f: write_f.write(json.dumps(dict(foo="bar"), ensure_ascii=False)) patcher = mock.patch("io.open") mock_open = patcher.start() mock_open.side_effect = OSError("OSError") self.addCleanup(patcher.stop) job_doc = mjob.JobDocument("job", "kernel", "git_branch") build_doc = utils.build._traverse_build_dir( build_dir, job_doc, errors, {}) finally: shutil.rmtree(temp_dir, ignore_errors=True) self.assertIsNotNone(errors) self.assertIsNone(build_doc) self.assertListEqual([500], errors.keys())
def test_job_correct_status(self): job_doc = modj.JobDocument("job", "kernel", "git-branch") job_doc.status = "FAIL" self.assertEqual(job_doc.status, "FAIL") job_doc.status = "BUILD" self.assertEqual(job_doc.status, "BUILD") job_doc.status = "PASS" self.assertEqual(job_doc.status, "PASS")
def test_job_document_private(self): # By default, jobs are public. job_doc = modj.JobDocument("job", "kernel", "git-branch") self.assertFalse(job_doc.private) job_doc.private = True self.assertTrue(job_doc.private)
def test_update_job_doc_no_defconfig(self): job_doc = mjob.JobDocument("job", "kernel", "branch") utils.build._update_job_doc(job_doc, None, "PASS", None, self.db) self.assertIsInstance(job_doc, mjob.JobDocument) self.assertIsNotNone(job_doc.git_branch) self.assertIsNone(job_doc.id) self.assertIsNone(job_doc.git_commit) self.assertIsNone(job_doc.git_describe) self.assertIsNone(job_doc.git_url)
def test_traverse_build_dir_data_sizes(self, get_size): get_size.return_value = [("system_map_size", 1024), ("modules_size", 1025), ("build_log_size", 1026), ("kernel_config_size", 1027), ("kernel_image_size", 1028), ("vmlinux_file_size", 1029)] build_data = { "arch": "arm", "build_environment": "build_environment", "git_url": "git://git.example.org", "git_branch": "test/branch", "git_describe": "vfoo.bar", "git_commit": "1234567890", "defconfig": "defoo_confbar", "kernel_image": "zImage", "kernel_config": "kernel.config", "modules": "modules.tar", "modules_dir": "foo/bar", "build_log": "file.log", "vmlinux_file": "vmlinux", "system_map": "System.map" } job_doc = mjob.JobDocument("job", "kernel", "git_branch") job_doc.created_on = "job_date" try: errors = {} temp_dir = tempfile.mkdtemp() build_dir = os.path.join(temp_dir, "build_dir") os.mkdir(build_dir) with io.open(os.path.join(build_dir, "bmeta.json"), mode="w") as f: f.write(json.dumps(build_data, ensure_ascii=False)) build_doc = utils.build._traverse_build_dir( build_dir, job_doc, errors, {}) finally: shutil.rmtree(temp_dir, ignore_errors=True) self.assertIsInstance(build_doc, mbuild.BuildDocument) self.assertEqual(build_doc.system_map_size, 1024) self.assertEqual(build_doc.modules_size, 1025) self.assertEqual(build_doc.build_log_size, 1026) self.assertEqual(build_doc.kernel_config_size, 1027) self.assertEqual(build_doc.kernel_image_size, 1028) self.assertEqual(build_doc.vmlinux_file_size, 1029)
def test_traverse_build_dir_with_valid_data(self): build_data = { "arch": "arm", "build_environment": "build_environment", "git_url": "git://git.example.org", "git_branch": "branch", "git_describe": "vfoo.bar", "git_commit": "1234567890", "defconfig": "defoo_confbar", "kernel_image": "zImage", "kernel_config": "kernel.config", "modules_dir": "foo/bar", "build_log": "file.log", "vmlinux_file": "vmlinux", "vmlinux_bss_size": 1024, "vmlinux_data_size": 1024, "vmlinux_file_size": 1024, "vmlinux_text_size": 1024, } job_doc = mjob.JobDocument("job", "kernel", "branch") job_doc.created_on = "job_date" try: errors = {} temp_dir = tempfile.mkdtemp() build_dir = os.path.join(temp_dir, "build_dir") os.mkdir(build_dir) with io.open(os.path.join(build_dir, "bmeta.json"), mode="w") as f: f.write(json.dumps(build_data, ensure_ascii=False)) build_doc = utils.build._traverse_build_dir( build_dir, job_doc, errors, {}) finally: shutil.rmtree(temp_dir, ignore_errors=True) self.assertIsNotNone(build_doc) self.assertIsInstance(build_doc, mbuild.BuildDocument) self.assertEqual("job", build_doc.job) self.assertEqual("kernel", build_doc.kernel) self.assertEqual("kernel", build_doc.build_type) self.assertEqual(build_doc.vmlinux_text_size, 1024) self.assertEqual(build_doc.vmlinux_bss_size, 1024) self.assertEqual(build_doc.vmlinux_data_size, 1024) self.assertEqual(build_doc.vmlinux_file_size, 1024)
def test_traverse_buld_dir_with_empty_file(self): try: errors = {} temp_dir = tempfile.mkdtemp() build_dir = os.path.join(temp_dir, "build_dir") os.mkdir(build_dir) io.open(os.path.join(build_dir, "build.json"), mode="w") job_doc = mjob.JobDocument("job", "kernel", "git_branch") build_doc = utils.build._traverse_build_dir( build_dir, job_doc, errors, {}) finally: shutil.rmtree(temp_dir, ignore_errors=True) self.assertIsNotNone(errors) self.assertIsNone(build_doc) self.assertListEqual([500], errors.keys())
def test_traverse_buld_dir_with_json_error(self): try: errors = {} temp_dir = tempfile.mkdtemp() build_dir = os.path.join(temp_dir, "build_dir") os.mkdir(build_dir) with io.open(os.path.join(build_dir, "build.json"), mode="w") as f: f.write(u"a string of text") job_doc = mjob.JobDocument("job", "kernel", "branch") job_doc.created_on = "job_date" build_doc = utils.build._traverse_build_dir( build_dir, job_doc, errors, {}) finally: shutil.rmtree(temp_dir, ignore_errors=True) self.assertIsNotNone(errors) self.assertIsNone(build_doc) self.assertListEqual([500], errors.keys())
def _get_or_create_job(job, kernel, git_branch, database, db_options): """Get or create a job in the database. :param job: The name of the job. :type job: str :param kernel: The name of the kernel. :type kernel: str :param database: The mongodb database connection. :param db_options: The database connection options. :type db_options: dict :return a 3-tuple: return value, job document and job ID. """ ret_val = 201 job_doc = None job_id = None redis_conn = redisdb.get_db_connection(db_options) # We might be importing build in parallel through multi-processes. # Keep a lock here when looking for a job or we might end up with # multiple job creations. lock_key = "build-import-{}-{}-{}".format(job, kernel, git_branch) with redis.lock.Lock(redis_conn, lock_key, timeout=5): p_doc = utils.db.find_one2( database[models.JOB_COLLECTION], { models.JOB_KEY: job, models.KERNEL_KEY: kernel, models.GIT_BRANCH_KEY: git_branch }) if p_doc: job_doc = mjob.JobDocument.from_json(p_doc) job_id = job_doc.id else: job_doc = mjob.JobDocument(job, kernel, git_branch) job_doc.status = models.BUILD_STATUS job_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) ret_val, job_id = utils.db.save(database, job_doc) job_doc.id = job_id return ret_val, job_doc, job_id
def test_update_job_doc(self): job_doc = mjob.JobDocument("job", "kernel", "branch") build_doc = mbuild.BuildDocument("job", "kernel", "defconfig", "branch") build_doc.git_branch = "branch" build_doc.git_commit = "1234567890" build_doc.git_describe = "kernel.version" build_doc.git_url = "git://url.git" utils.build._update_job_doc(job_doc, "job_id", "PASS", build_doc, self.db) self.assertIsInstance(job_doc, mjob.JobDocument) self.assertIsNotNone(job_doc.id) self.assertIsNotNone(job_doc.git_branch) self.assertIsNotNone(job_doc.git_commit) self.assertIsNotNone(job_doc.git_describe) self.assertIsNotNone(job_doc.git_url) self.assertEqual("job_id", job_doc.id) self.assertEqual("1234567890", job_doc.git_commit) self.assertEqual("kernel.version", job_doc.git_describe) self.assertEqual("git://url.git", job_doc.git_url)
def test_job_document_to_dict(self): job_doc = modj.JobDocument("job", "kernel", "git-branch") job_doc.id = "job" job_doc.created_on = "now" job_doc.status = "PASS" job_doc.git_commit = "1234" job_doc.git_url = "git-url" job_doc.git_describe = "git-describe" job_doc.git_describe_v = "git-describe-v" job_doc.kernel_version = "kernel-version" job_doc.compiler = "gcc" job_doc.compiler_version = "4.7.3" job_doc.compiler_version_ext = "gcc 4.7.3" job_doc.compiler_version_full = "gcc version 4.7.3" job_doc.cross_compile = "cross-compile" expected = { "_id": "job", "kernel": "kernel", "job": "job", "private": False, "created_on": "now", "status": "PASS", "version": "1.1", "git_commit": "1234", "git_url": "git-url", "git_branch": "git-branch", "git_describe": "git-describe", "git_describe_v": "git-describe-v", "kernel_version": "kernel-version", "compiler": "gcc", "compiler_version": "4.7.3", "compiler_version_ext": "gcc 4.7.3", "compiler_version_full": "gcc version 4.7.3", "cross_compile": "cross-compile" } self.assertEqual(job_doc.to_dict(), expected)
def test_job_document_date_serialization(self): now = datetime.datetime.now(tz=tz_util.utc) job_doc = modj.JobDocument("job", "kernel", "git-branch") job_doc.created_on = now self.assertIsInstance(job_doc.created_on, datetime.datetime) json_obj = { "_id": "job", "job": "job", "kernel": "kernel", "git_branch": "git-branch", "created_on": now } json_deserialized = json_util.loads(json_util.dumps(json_obj)) new_job = modj.JobDocument.from_json(json_deserialized) self.assertIsInstance(new_job.created_on, datetime.datetime) # During the deserialization process, some microseconds are lost. self.assertLessEqual( (new_job.created_on - job_doc.created_on).total_seconds(), 0)
def test_job_document_collection(self): job_doc = modj.JobDocument("job", "kernel", "git-branch") self.assertEqual(job_doc.collection, "job")
def test_job_documet_valid_instance(self): job_doc = modj.JobDocument("job", "kernel", "git-branch") self.assertIsInstance(job_doc, modb.BaseDocument) self.assertIsInstance(job_doc, modj.JobDocument)