def test_traverse_buld_dir_with_oserror(self):
try:
errors = {}
temp_dir = tempfile.mkdtemp()
build_dir = os.path.join(temp_dir, "build_dir")
os.mkdir(build_dir)
build_file = os.path.join(build_dir, "build.json")
with io.open(build_file, mode="w") as write_f:
write_f.write(json.dumps(dict(foo="bar"), ensure_ascii=False))
patcher = mock.patch("io.open")
mock_open = patcher.start()
mock_open.side_effect = OSError("OSError")
self.addCleanup(patcher.stop)
job_doc = mjob.JobDocument("job", "kernel", "git_branch")
build_doc = utils.build._traverse_build_dir(
build_dir, job_doc, errors, {})
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
self.assertIsNotNone(errors)
self.assertIsNone(build_doc)
self.assertListEqual([500], errors.keys())
def test_job_correct_status(self):
job_doc = modj.JobDocument("job", "kernel", "git-branch")
job_doc.status = "FAIL"
self.assertEqual(job_doc.status, "FAIL")
job_doc.status = "BUILD"
self.assertEqual(job_doc.status, "BUILD")
job_doc.status = "PASS"
self.assertEqual(job_doc.status, "PASS")
def test_job_document_private(self):
# By default, jobs are public.
job_doc = modj.JobDocument("job", "kernel", "git-branch")
self.assertFalse(job_doc.private)
job_doc.private = True
self.assertTrue(job_doc.private)
def test_update_job_doc_no_defconfig(self):
job_doc = mjob.JobDocument("job", "kernel", "branch")
utils.build._update_job_doc(job_doc, None, "PASS", None, self.db)
self.assertIsInstance(job_doc, mjob.JobDocument)
self.assertIsNotNone(job_doc.git_branch)
self.assertIsNone(job_doc.id)
self.assertIsNone(job_doc.git_commit)
self.assertIsNone(job_doc.git_describe)
self.assertIsNone(job_doc.git_url)
def test_traverse_build_dir_data_sizes(self, get_size):
get_size.return_value = [("system_map_size", 1024),
("modules_size", 1025),
("build_log_size", 1026),
("kernel_config_size", 1027),
("kernel_image_size", 1028),
("vmlinux_file_size", 1029)]
build_data = {
"arch": "arm",
"build_environment": "build_environment",
"git_url": "git://git.example.org",
"git_branch": "test/branch",
"git_describe": "vfoo.bar",
"git_commit": "1234567890",
"defconfig": "defoo_confbar",
"kernel_image": "zImage",
"kernel_config": "kernel.config",
"modules": "modules.tar",
"modules_dir": "foo/bar",
"build_log": "file.log",
"vmlinux_file": "vmlinux",
"system_map": "System.map"
}
job_doc = mjob.JobDocument("job", "kernel", "git_branch")
job_doc.created_on = "job_date"
try:
errors = {}
temp_dir = tempfile.mkdtemp()
build_dir = os.path.join(temp_dir, "build_dir")
os.mkdir(build_dir)
with io.open(os.path.join(build_dir, "bmeta.json"), mode="w") as f:
f.write(json.dumps(build_data, ensure_ascii=False))
build_doc = utils.build._traverse_build_dir(
build_dir, job_doc, errors, {})
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
self.assertIsInstance(build_doc, mbuild.BuildDocument)
self.assertEqual(build_doc.system_map_size, 1024)
self.assertEqual(build_doc.modules_size, 1025)
self.assertEqual(build_doc.build_log_size, 1026)
self.assertEqual(build_doc.kernel_config_size, 1027)
self.assertEqual(build_doc.kernel_image_size, 1028)
self.assertEqual(build_doc.vmlinux_file_size, 1029)
def test_traverse_build_dir_with_valid_data(self):
build_data = {
"arch": "arm",
"build_environment": "build_environment",
"git_url": "git://git.example.org",
"git_branch": "branch",
"git_describe": "vfoo.bar",
"git_commit": "1234567890",
"defconfig": "defoo_confbar",
"kernel_image": "zImage",
"kernel_config": "kernel.config",
"modules_dir": "foo/bar",
"build_log": "file.log",
"vmlinux_file": "vmlinux",
"vmlinux_bss_size": 1024,
"vmlinux_data_size": 1024,
"vmlinux_file_size": 1024,
"vmlinux_text_size": 1024,
}
job_doc = mjob.JobDocument("job", "kernel", "branch")
job_doc.created_on = "job_date"
try:
errors = {}
temp_dir = tempfile.mkdtemp()
build_dir = os.path.join(temp_dir, "build_dir")
os.mkdir(build_dir)
with io.open(os.path.join(build_dir, "bmeta.json"), mode="w") as f:
f.write(json.dumps(build_data, ensure_ascii=False))
build_doc = utils.build._traverse_build_dir(
build_dir, job_doc, errors, {})
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
self.assertIsNotNone(build_doc)
self.assertIsInstance(build_doc, mbuild.BuildDocument)
self.assertEqual("job", build_doc.job)
self.assertEqual("kernel", build_doc.kernel)
self.assertEqual("kernel", build_doc.build_type)
self.assertEqual(build_doc.vmlinux_text_size, 1024)
self.assertEqual(build_doc.vmlinux_bss_size, 1024)
self.assertEqual(build_doc.vmlinux_data_size, 1024)
self.assertEqual(build_doc.vmlinux_file_size, 1024)
def test_traverse_buld_dir_with_empty_file(self):
try:
errors = {}
temp_dir = tempfile.mkdtemp()
build_dir = os.path.join(temp_dir, "build_dir")
os.mkdir(build_dir)
io.open(os.path.join(build_dir, "build.json"), mode="w")
job_doc = mjob.JobDocument("job", "kernel", "git_branch")
build_doc = utils.build._traverse_build_dir(
build_dir, job_doc, errors, {})
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
self.assertIsNotNone(errors)
self.assertIsNone(build_doc)
self.assertListEqual([500], errors.keys())
def test_traverse_buld_dir_with_json_error(self):
try:
errors = {}
temp_dir = tempfile.mkdtemp()
build_dir = os.path.join(temp_dir, "build_dir")
os.mkdir(build_dir)
with io.open(os.path.join(build_dir, "build.json"), mode="w") as f:
f.write(u"a string of text")
job_doc = mjob.JobDocument("job", "kernel", "branch")
job_doc.created_on = "job_date"
build_doc = utils.build._traverse_build_dir(
build_dir, job_doc, errors, {})
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
self.assertIsNotNone(errors)
self.assertIsNone(build_doc)
self.assertListEqual([500], errors.keys())
def _get_or_create_job(job, kernel, git_branch, database, db_options):
"""Get or create a job in the database.
:param job: The name of the job.
:type job: str
:param kernel: The name of the kernel.
:type kernel: str
:param database: The mongodb database connection.
:param db_options: The database connection options.
:type db_options: dict
:return a 3-tuple: return value, job document and job ID.
"""
ret_val = 201
job_doc = None
job_id = None
redis_conn = redisdb.get_db_connection(db_options)
# We might be importing build in parallel through multi-processes.
# Keep a lock here when looking for a job or we might end up with
# multiple job creations.
lock_key = "build-import-{}-{}-{}".format(job, kernel, git_branch)
with redis.lock.Lock(redis_conn, lock_key, timeout=5):
p_doc = utils.db.find_one2(
database[models.JOB_COLLECTION], {
models.JOB_KEY: job,
models.KERNEL_KEY: kernel,
models.GIT_BRANCH_KEY: git_branch
})
if p_doc:
job_doc = mjob.JobDocument.from_json(p_doc)
job_id = job_doc.id
else:
job_doc = mjob.JobDocument(job, kernel, git_branch)
job_doc.status = models.BUILD_STATUS
job_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
ret_val, job_id = utils.db.save(database, job_doc)
job_doc.id = job_id
return ret_val, job_doc, job_id
def test_update_job_doc(self):
job_doc = mjob.JobDocument("job", "kernel", "branch")
build_doc = mbuild.BuildDocument("job", "kernel", "defconfig",
"branch")
build_doc.git_branch = "branch"
build_doc.git_commit = "1234567890"
build_doc.git_describe = "kernel.version"
build_doc.git_url = "git://url.git"
utils.build._update_job_doc(job_doc, "job_id", "PASS", build_doc,
self.db)
self.assertIsInstance(job_doc, mjob.JobDocument)
self.assertIsNotNone(job_doc.id)
self.assertIsNotNone(job_doc.git_branch)
self.assertIsNotNone(job_doc.git_commit)
self.assertIsNotNone(job_doc.git_describe)
self.assertIsNotNone(job_doc.git_url)
self.assertEqual("job_id", job_doc.id)
self.assertEqual("1234567890", job_doc.git_commit)
self.assertEqual("kernel.version", job_doc.git_describe)
self.assertEqual("git://url.git", job_doc.git_url)
def test_job_document_to_dict(self):
job_doc = modj.JobDocument("job", "kernel", "git-branch")
job_doc.id = "job"
job_doc.created_on = "now"
job_doc.status = "PASS"
job_doc.git_commit = "1234"
job_doc.git_url = "git-url"
job_doc.git_describe = "git-describe"
job_doc.git_describe_v = "git-describe-v"
job_doc.kernel_version = "kernel-version"
job_doc.compiler = "gcc"
job_doc.compiler_version = "4.7.3"
job_doc.compiler_version_ext = "gcc 4.7.3"
job_doc.compiler_version_full = "gcc version 4.7.3"
job_doc.cross_compile = "cross-compile"
expected = {
"_id": "job",
"kernel": "kernel",
"job": "job",
"private": False,
"created_on": "now",
"status": "PASS",
"version": "1.1",
"git_commit": "1234",
"git_url": "git-url",
"git_branch": "git-branch",
"git_describe": "git-describe",
"git_describe_v": "git-describe-v",
"kernel_version": "kernel-version",
"compiler": "gcc",
"compiler_version": "4.7.3",
"compiler_version_ext": "gcc 4.7.3",
"compiler_version_full": "gcc version 4.7.3",
"cross_compile": "cross-compile"
}
self.assertEqual(job_doc.to_dict(), expected)
def test_job_document_date_serialization(self):
now = datetime.datetime.now(tz=tz_util.utc)
job_doc = modj.JobDocument("job", "kernel", "git-branch")
job_doc.created_on = now
self.assertIsInstance(job_doc.created_on, datetime.datetime)
json_obj = {
"_id": "job",
"job": "job",
"kernel": "kernel",
"git_branch": "git-branch",
"created_on": now
}
json_deserialized = json_util.loads(json_util.dumps(json_obj))
new_job = modj.JobDocument.from_json(json_deserialized)
self.assertIsInstance(new_job.created_on, datetime.datetime)
# During the deserialization process, some microseconds are lost.
self.assertLessEqual(
(new_job.created_on - job_doc.created_on).total_seconds(), 0)
def test_job_document_collection(self):
job_doc = modj.JobDocument("job", "kernel", "git-branch")
self.assertEqual(job_doc.collection, "job")
def test_job_documet_valid_instance(self):
job_doc = modj.JobDocument("job", "kernel", "git-branch")
self.assertIsInstance(job_doc, modb.BaseDocument)
self.assertIsInstance(job_doc, modj.JobDocument)
