def test_metastore(self):
field = TestCase._meta.get_field('metadata')
level = '1.3.5.1'
# artificially inflate results to represent a set of kernel messages
results = {
'definition': 'lava',
'case': 'unit-test',
# list of numbers, generates a much longer YAML string than just the count
'extra': range(int(field.max_length / 2)),
'result': 'pass'
}
stub = "%s-%s-%s.yaml" % (results['definition'], results['case'], level)
job = TestJob.from_yaml_and_user(
self.factory.make_job_yaml(), self.user)
meta_filename = os.path.join(job.output_dir, 'metadata', stub)
filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % (job.output_dir,
job.id, level.split('.')[0],
level, results['definition'])
mkdir(os.path.dirname(filename))
if os.path.exists(meta_filename):
# isolate from other unit tests
os.unlink(meta_filename)
self.assertEqual(meta_filename, create_metadata_store(results, job, level))
self.assertTrue(map_scanned_results(results, job, meta_filename))
self.assertEqual(TestCase.objects.filter(name='unit-test').count(), 1)
test_data = yaml.load(TestCase.objects.filter(name='unit-test')[0].metadata, Loader=yaml.CLoader)
self.assertEqual(test_data['extra'], meta_filename)
self.assertTrue(os.path.exists(meta_filename))
with open(test_data['extra'], 'r') as extra_file:
data = yaml.load(extra_file, Loader=yaml.CLoader)
self.assertIsNotNone(data)
os.unlink(meta_filename)
shutil.rmtree(job.output_dir)
def _handle_end(self, hostname, action, msg): # pylint: disable=unused-argument
try:
job_id = int(msg[2])
error_msg = msg[3]
compressed_description = msg[4]
except (IndexError, ValueError):
self.logger.error("Invalid message from <%s> '%s'", hostname, msg)
return
try:
job = TestJob.objects.get(id=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%d] Unknown job", job_id)
# ACK even if the job is unknown to let the dispatcher
# forget about it
send_multipart_u(self.controler, [hostname, 'END_OK', str(job_id)])
return
filename = os.path.join(job.output_dir, 'description.yaml')
# If description.yaml already exists: a END was already received
if os.path.exists(filename):
self.logger.info("[%d] %s => END (duplicated), skipping", job_id,
hostname)
else:
if compressed_description:
self.logger.info("[%d] %s => END", job_id, hostname)
else:
self.logger.info(
"[%d] %s => END (lava-run crashed, mark job as INCOMPLETE)",
job_id, hostname)
with transaction.atomic():
# TODO: find a way to lock actual_device
job = TestJob.objects.select_for_update() \
.get(id=job_id)
job.go_state_finished(TestJob.HEALTH_INCOMPLETE)
if error_msg:
self.logger.error("[%d] Error: %s", job_id, error_msg)
job.failure_comment = error_msg
job.save()
# Create description.yaml even if it's empty
# Allows to know when END messages are duplicated
try:
# Create the directory if it was not already created
mkdir(os.path.dirname(filename))
# TODO: check that compressed_description is not ""
description = lzma.decompress(compressed_description)
with open(filename, 'w') as f_description:
f_description.write(description.decode("utf-8"))
if description:
parse_job_description(job)
except (OSError, lzma.LZMAError) as exc:
self.logger.error("[%d] Unable to dump 'description.yaml'",
job_id)
self.logger.exception("[%d] %s", job_id, exc)
# ACK the job and mark the dispatcher as alive
send_multipart_u(self.controler, [hostname, 'END_OK', str(job_id)])
self.dispatcher_alive(hostname)
def _handle_end(self, hostname, action, msg): # pylint: disable=unused-argument
try:
job_id = int(msg[2])
error_msg = msg[3]
compressed_description = msg[4]
except (IndexError, ValueError):
self.logger.error("Invalid message from <%s> '%s'", hostname, msg)
return
try:
job = TestJob.objects.get(id=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%d] Unknown job", job_id)
# ACK even if the job is unknown to let the dispatcher
# forget about it
send_multipart_u(self.controler, [hostname, 'END_OK', str(job_id)])
return
filename = os.path.join(job.output_dir, 'description.yaml')
# If description.yaml already exists: a END was already received
if os.path.exists(filename):
self.logger.info("[%d] %s => END (duplicated), skipping", job_id, hostname)
else:
if compressed_description:
self.logger.info("[%d] %s => END", job_id, hostname)
else:
self.logger.info("[%d] %s => END (lava-run crashed, mark job as INCOMPLETE)",
job_id, hostname)
with transaction.atomic():
# TODO: find a way to lock actual_device
job = TestJob.objects.select_for_update() \
.get(id=job_id)
job.go_state_finished(TestJob.HEALTH_INCOMPLETE)
if error_msg:
self.logger.error("[%d] Error: %s", job_id, error_msg)
job.failure_comment = error_msg
job.save()
# Create description.yaml even if it's empty
# Allows to know when END messages are duplicated
try:
# Create the directory if it was not already created
mkdir(os.path.dirname(filename))
# TODO: check that compressed_description is not ""
description = lzma.decompress(compressed_description)
with open(filename, 'w') as f_description:
f_description.write(description.decode("utf-8"))
if description:
parse_job_description(job)
except (IOError, lzma.LZMAError) as exc:
self.logger.error("[%d] Unable to dump 'description.yaml'",
job_id)
self.logger.exception("[%d] %s", job_id, exc)
# ACK the job and mark the dispatcher as alive
send_multipart_u(self.controler, [hostname, 'END_OK', str(job_id)])
self.dispatcher_alive(hostname)
def save_job_config(self, job, worker, device_cfg, options):
output_dir = job.output_dir
mkdir(output_dir)
with open(os.path.join(output_dir, "job.yaml"), "w") as f_out:
f_out.write(self.export_definition(job))
with contextlib.suppress(IOError):
shutil.copy(options["env"], os.path.join(output_dir, "env.yaml"))
with contextlib.suppress(IOError):
shutil.copy(options["env_dut"], os.path.join(output_dir, "env.dut.yaml"))
with contextlib.suppress(IOError):
shutil.copy(os.path.join(options["dispatchers_config"], "%s.yaml" % worker.hostname),
os.path.join(output_dir, "dispatcher.yaml"))
with open(os.path.join(output_dir, "device.yaml"), "w") as f_out:
yaml.dump(device_cfg, f_out)
def save_job_config(self, job, device_cfg, env_str, env_dut_str, dispatcher_cfg):
output_dir = job.output_dir
mkdir(output_dir)
with open(os.path.join(output_dir, "job.yaml"), "w") as f_out:
f_out.write(self.export_definition(job))
with open(os.path.join(output_dir, "device.yaml"), "w") as f_out:
yaml.dump(device_cfg, f_out)
if env_str:
with open(os.path.join(output_dir, "env.yaml"), "w") as f_out:
f_out.write(env_str)
if env_dut_str:
with open(os.path.join(output_dir, "env.dut.yaml"), "w") as f_out:
f_out.write(env_dut_str)
if dispatcher_cfg:
with open(os.path.join(output_dir, "dispatcher.yaml"), "w") as f_out:
f_out.write(dispatcher_cfg)
def handle(self, *_, **options):
base_dir = "/var/lib/lava-server/default/media/job-output/"
len_base_dir = len(base_dir)
jobs = TestJob.objects.all().order_by("id")
self.stdout.write("Browsing all jobs")
for job in jobs:
self.stdout.write(
"* %d {%s => %s}" %
(job.id, "job-%d" % job.id, job.output_dir[len_base_dir:]))
mkdir(os.path.dirname(job.output_dir))
old_dir = base_dir + "job-%d" % job.id
if not os.path.exists(old_dir):
self.stdout.write(" -> no output directory")
continue
os.rename(old_dir, job.output_dir)
def save_job_config(self, job, worker, device_cfg, options):
output_dir = job.output_dir
mkdir(output_dir)
with open(os.path.join(output_dir, "job.yaml"), "w") as f_out:
f_out.write(self.export_definition(job))
with contextlib.suppress(IOError):
shutil.copy(options["env"], os.path.join(output_dir, "env.yaml"))
with contextlib.suppress(IOError):
shutil.copy(options["env_dut"],
os.path.join(output_dir, "env.dut.yaml"))
with contextlib.suppress(IOError):
shutil.copy(
os.path.join(options["dispatchers_config"],
"%s.yaml" % worker.hostname),
os.path.join(output_dir, "dispatcher.yaml"))
with open(os.path.join(output_dir, "device.yaml"), "w") as f_out:
yaml.dump(device_cfg, f_out)
def test_metastore(self):
field = TestCase._meta.get_field("metadata")
level = "1.3.5.1"
# artificially inflate results to represent a set of kernel messages
results = {
"definition": "lava",
"case": "unit-test",
"level": level,
# list of numbers, generates a much longer YAML string than just the count
"extra": range(int(field.max_length / 2)),
"result": "pass",
}
stub = "%s-%s-%s.yaml" % (results["definition"], results["case"],
level)
job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(),
self.user)
meta_filename = os.path.join(job.output_dir, "metadata", stub)
filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % (
job.output_dir,
job.id,
level.split(".")[0],
level,
results["definition"],
)
mkdir(os.path.dirname(filename))
if os.path.exists(meta_filename):
# isolate from other unit tests
os.unlink(meta_filename)
self.assertEqual(meta_filename, create_metadata_store(results, job))
ret = map_scanned_results(results, job, {}, meta_filename)
self.assertIsNotNone(ret)
ret.save()
self.assertEqual(TestCase.objects.filter(name="unit-test").count(), 1)
test_data = yaml.load( # nosec - unit test
TestCase.objects.filter(name="unit-test")[0].metadata,
Loader=yaml.CLoader)
self.assertEqual(test_data["extra"], meta_filename)
self.assertTrue(os.path.exists(meta_filename))
with open(test_data["extra"], "r") as extra_file:
data = yaml.load(extra_file,
Loader=yaml.CLoader) # nosec - unit test
self.assertIsNotNone(data)
os.unlink(meta_filename)
shutil.rmtree(job.output_dir)
def handle(self, *_, **options):
base_dir = "/var/lib/lava-server/default/media/job-output/"
len_base_dir = len(base_dir)
jobs = TestJob.objects.all().order_by("id")
self.stdout.write("Browsing all jobs")
start = 0
while True:
count = 0
for job in jobs[start:start + 100]:
count += 1
old_path = os.path.join(settings.MEDIA_ROOT, "job-output",
"job-%s" % job.id)
date_path = os.path.join(
settings.MEDIA_ROOT,
"job-output",
"%02d" % job.submit_time.year,
"%02d" % job.submit_time.month,
"%02d" % job.submit_time.day,
str(job.id),
)
if not os.path.exists(old_path):
self.stdout.write("* %d skip" % job.id)
continue
self.stdout.write("* %d {%s => %s}" %
(job.id, old_path[len_base_dir:],
date_path[len_base_dir:]))
if not options["dry_run"]:
mkdir(os.path.dirname(date_path))
if not os.path.exists(old_path):
self.stdout.write(" -> no output directory")
continue
os.rename(old_path, date_path)
start += count
if count == 0:
break
if options["slow"]:
self.stdout.write("sleeping 2s...")
time.sleep(2)
def test_metastore(self):
field = TestCase._meta.get_field('metadata')
level = '1.3.5.1'
# artificially inflate results to represent a set of kernel messages
results = {
'definition': 'lava',
'case': 'unit-test',
'level': level,
# list of numbers, generates a much longer YAML string than just the count
'extra': range(int(field.max_length / 2)),
'result': 'pass'
}
stub = "%s-%s-%s.yaml" % (results['definition'], results['case'],
level)
job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(),
self.user)
meta_filename = os.path.join(job.output_dir, 'metadata', stub)
filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % (
job.output_dir, job.id, level.split('.')[0], level,
results['definition'])
mkdir(os.path.dirname(filename))
if os.path.exists(meta_filename):
# isolate from other unit tests
os.unlink(meta_filename)
self.assertEqual(meta_filename, create_metadata_store(results, job))
ret = map_scanned_results(results, job, {}, meta_filename)
self.assertIsNotNone(ret)
ret.save()
self.assertEqual(TestCase.objects.filter(name='unit-test').count(), 1)
test_data = yaml.load(
TestCase.objects.filter(name='unit-test')[0].metadata,
Loader=yaml.CLoader)
self.assertEqual(test_data['extra'], meta_filename)
self.assertTrue(os.path.exists(meta_filename))
with open(test_data['extra'], 'r') as extra_file:
data = yaml.load(extra_file, Loader=yaml.CLoader)
self.assertIsNotNone(data)
os.unlink(meta_filename)
shutil.rmtree(job.output_dir)
def handle(self, *_, **options):
base_dir = "/var/lib/lava-server/default/media/job-output/"
len_base_dir = len(base_dir)
jobs = TestJob.objects.all().order_by("id")
self.stdout.write("Browsing all jobs")
start = 0
while True:
count = 0
for job in jobs[start:start + 100]:
count += 1
old_path = os.path.join(settings.MEDIA_ROOT, 'job-output', 'job-%s' % job.id)
date_path = os.path.join(settings.MEDIA_ROOT, 'job-output',
"%02d" % job.submit_time.year,
"%02d" % job.submit_time.month,
"%02d" % job.submit_time.day,
str(job.id))
if not os.path.exists(old_path):
self.stdout.write("* %d skip" % job.id)
continue
self.stdout.write("* %d {%s => %s}" % (job.id,
old_path[len_base_dir:],
date_path[len_base_dir:]))
if not options["dry_run"]:
mkdir(os.path.dirname(date_path))
if not os.path.exists(old_path):
self.stdout.write(" -> no output directory")
continue
os.rename(old_path, date_path)
start += count
if count == 0:
break
if options["slow"]:
self.stdout.write("sleeping 2s...")
time.sleep(2)
def handle(self, *_, **options):
base_dir = "/var/lib/lava-server/default/media/job-output/"
len_base_dir = len(base_dir)
jobs = TestJob.objects.all().order_by("id")
self.stdout.write("Browsing all jobs")
for job in jobs:
old_path = os.path.join(settings.MEDIA_ROOT, 'job-output', 'job-%s' % job.id)
date_path = os.path.join(settings.MEDIA_ROOT, 'job-output',
"%02d" % job.submit_time.year,
"%02d" % job.submit_time.month,
"%02d" % job.submit_time.day,
str(job.id))
if not os.path.exists(old_path):
continue
self.stdout.write("* %d {%s => %s}" % (job.id,
old_path[len_base_dir:],
date_path[len_base_dir:]))
mkdir(os.path.dirname(date_path))
if not os.path.exists(old_path):
self.stdout.write(" -> no output directory")
continue
os.rename(old_path, date_path)
def logging_socket(self):
msg = self.log_socket.recv_multipart()
try:
(job_id, message) = (u(m) for m in msg) # pylint: disable=unbalanced-tuple-unpacking
except ValueError:
# do not let a bad message stop the master.
self.logger.error("[POLL] failed to parse log message, skipping: %s", msg)
return
try:
scanned = yaml.load(message, Loader=yaml.CLoader)
except yaml.YAMLError:
self.logger.error("[%s] data are not valid YAML, dropping", job_id)
return
# Look for "results" level
try:
message_lvl = scanned["lvl"]
message_msg = scanned["msg"]
except TypeError:
self.logger.error("[%s] not a dictionary, dropping", job_id)
return
except KeyError:
self.logger.error(
"[%s] invalid log line, missing \"lvl\" or \"msg\" keys: %s",
job_id, message)
return
# Find the handler (if available)
if job_id not in self.jobs:
# Query the database for the job
try:
job = TestJob.objects.get(id=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] unknown job id", job_id)
return
self.logger.info("[%s] receiving logs from a new job", job_id)
# Create the sub directories (if needed)
mkdir(job.output_dir)
self.jobs[job_id] = JobHandler(job)
if message_lvl == "results":
try:
job = TestJob.objects.get(pk=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] unknown job id", job_id)
return
meta_filename = create_metadata_store(message_msg, job)
new_test_case = map_scanned_results(results=message_msg, job=job,
meta_filename=meta_filename)
if new_test_case is None:
self.logger.warning(
"[%s] unable to map scanned results: %s",
job_id, message)
else:
self.test_cases.append(new_test_case)
# Look for lava.job result
if message_msg.get("definition") == "lava" and message_msg.get("case") == "job":
# Flush cached test cases
self.flush_test_cases()
if message_msg.get("result") == "pass":
health = TestJob.HEALTH_COMPLETE
health_msg = "Complete"
else:
health = TestJob.HEALTH_INCOMPLETE
health_msg = "Incomplete"
self.logger.info("[%s] job status: %s", job_id, health_msg)
infrastructure_error = (message_msg.get("error_type") in ["Bug",
"Configuration",
"Infrastructure"])
if infrastructure_error:
self.logger.info("[%s] Infrastructure error", job_id)
# Update status.
with transaction.atomic():
# TODO: find a way to lock actual_device
job = TestJob.objects.select_for_update() \
.get(id=job_id)
job.go_state_finished(health, infrastructure_error)
job.save()
# Mark the file handler as used
self.jobs[job_id].last_usage = time.time()
# n.b. logging here would produce a log entry for every message in every job.
# The format is a list of dictionaries
message = "- %s" % message
# Write data
self.jobs[job_id].write(message)
def logging_socket(self, options):
msg = self.pull_socket.recv_multipart()
try:
(job_id, level, name, message) = msg # pylint: disable=unbalanced-tuple-unpacking
except ValueError:
# do not let a bad message stop the master.
self.logger.error("Failed to parse log message, skipping: %s", msg)
return
try:
scanned = yaml.load(message, Loader=yaml.CLoader)
except yaml.YAMLError:
self.logger.error("[%s] data are not valid YAML, dropping", job_id)
return
# Look for "results" level
try:
message_lvl = scanned["lvl"]
message_msg = scanned["msg"]
except KeyError:
self.logger.error(
"[%s] Invalid log line, missing \"lvl\" or \"msg\" keys: %s",
job_id, message)
return
# Clear filename
if '/' in level or '/' in name:
self.logger.error(
"[%s] Wrong level or name received, dropping the message",
job_id)
return
# Find the handler (if available)
if job_id in self.jobs:
if level != self.jobs[job_id].current_level:
# Close the old file handler
self.jobs[job_id].sub_log.close()
filename = os.path.join(self.jobs[job_id].output_dir,
"pipeline",
level.split('.')[0],
"%s-%s.yaml" % (level, name))
mkdir(os.path.dirname(filename))
self.current_level = level
self.jobs[job_id].sub_log = open(filename, 'a+')
else:
# Query the database for the job
try:
job = TestJob.objects.get(id=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] Unknown job id", job_id)
return
self.logger.info("[%s] Receiving logs from a new job", job_id)
filename = os.path.join(job.output_dir, "pipeline",
level.split('.')[0],
"%s-%s.yaml" % (level, name))
# Create the sub directories (if needed)
mkdir(os.path.dirname(filename))
self.jobs[job_id] = JobHandler(job, level, filename)
if message_lvl == "results":
try:
job = TestJob.objects.get(pk=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] Unknown job id", job_id)
return
meta_filename = create_metadata_store(message_msg, job, level)
ret = map_scanned_results(results=message_msg,
job=job,
meta_filename=meta_filename)
if not ret:
self.logger.warning("[%s] Unable to map scanned results: %s",
job_id, message)
# Mark the file handler as used
self.jobs[job_id].last_usage = time.time()
# n.b. logging here would produce a log entry for every message in every job.
# The format is a list of dictionaries
message = "- %s" % message
# Write data
self.jobs[job_id].write(message)
def logging_socket(self):
msg = self.log_socket.recv_multipart()
try:
(job_id, message) = (u(m) for m in msg) # pylint: disable=unbalanced-tuple-unpacking
except ValueError:
# do not let a bad message stop the master.
self.logger.error("[POLL] failed to parse log message, skipping: %s", msg)
return
try:
scanned = yaml.load(message, Loader=yaml.CLoader)
except yaml.YAMLError:
self.logger.error("[%s] data are not valid YAML, dropping", job_id)
return
# Look for "results" level
try:
message_lvl = scanned["lvl"]
message_msg = scanned["msg"]
except TypeError:
self.logger.error("[%s] not a dictionary, dropping", job_id)
return
except KeyError:
self.logger.error(
"[%s] invalid log line, missing \"lvl\" or \"msg\" keys: %s",
job_id, message)
return
# Find the handler (if available)
if job_id not in self.jobs:
# Query the database for the job
try:
job = TestJob.objects.get(id=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] unknown job id", job_id)
return
self.logger.info("[%s] receiving logs from a new job", job_id)
# Create the sub directories (if needed)
mkdir(job.output_dir)
self.jobs[job_id] = JobHandler(job)
# For 'event', send an event and log as 'debug'
if message_lvl == 'event':
self.logger.debug("[%s] event: %s", job_id, message_msg)
send_event(".event", "lavaserver", {"message": message_msg, "job": job_id})
message_lvl = "debug"
# For 'marker', save in the database and log as 'debug'
elif message_lvl == 'marker':
# TODO: save on the file system in case of lava-logs restart
m_type = message_msg.get("type")
case = message_msg.get("case")
if m_type is None or case is None:
self.logger.error("[%s] invalid marker: %s", job_id, message_msg)
return
self.jobs[job_id].markers.setdefault(case, {})[m_type] = self.jobs[job_id].line_count()
# This is in fact the previous line
self.jobs[job_id].markers[case][m_type] -= 1
self.logger.debug("[%s] marker: %s line: %s", job_id, message_msg, self.jobs[job_id].markers[case][m_type])
return
# Mark the file handler as used
self.jobs[job_id].last_usage = time.time()
# The format is a list of dictionaries
self.jobs[job_id].write("- %s" % message)
if message_lvl == "results":
try:
job = TestJob.objects.get(pk=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] unknown job id", job_id)
return
meta_filename = create_metadata_store(message_msg, job)
new_test_case = map_scanned_results(results=message_msg, job=job,
markers=self.jobs[job_id].markers,
meta_filename=meta_filename)
if new_test_case is None:
self.logger.warning(
"[%s] unable to map scanned results: %s",
job_id, message)
else:
self.test_cases.append(new_test_case)
# Look for lava.job result
if message_msg.get("definition") == "lava" and message_msg.get("case") == "job":
# Flush cached test cases
self.flush_test_cases()
if message_msg.get("result") == "pass":
health = TestJob.HEALTH_COMPLETE
health_msg = "Complete"
else:
health = TestJob.HEALTH_INCOMPLETE
health_msg = "Incomplete"
self.logger.info("[%s] job status: %s", job_id, health_msg)
infrastructure_error = (message_msg.get("error_type") in ["Bug",
"Configuration",
"Infrastructure"])
if infrastructure_error:
self.logger.info("[%s] Infrastructure error", job_id)
# Update status.
with transaction.atomic():
# TODO: find a way to lock actual_device
job = TestJob.objects.select_for_update() \
.get(id=job_id)
job.go_state_finished(health, infrastructure_error)
job.save()
def logging_socket(self, options):
msg = self.pull_socket.recv_multipart()
try:
(job_id, level, name, message) = msg # pylint: disable=unbalanced-tuple-unpacking
except ValueError:
# do not let a bad message stop the master.
self.logger.error("Failed to parse log message, skipping: %s", msg)
return
try:
scanned = yaml.load(message)
except yaml.YAMLError:
self.logger.error("[%s] data are not valid YAML, dropping", job_id)
return
# Look for "results" level
try:
message_lvl = scanned["lvl"]
message_msg = scanned["msg"]
except KeyError:
self.logger.error(
"[%s] Invalid log line, missing \"lvl\" or \"msg\" keys: %s",
job_id, message)
return
# Clear filename
if '/' in level or '/' in name:
self.logger.error("[%s] Wrong level or name received, dropping the message", job_id)
return
filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % (options['output_dir'],
job_id, level.split('.')[0],
level, name)
# Find the handler (if available)
if job_id in self.logs:
if filename != self.logs[job_id].filename:
# Close the old file handler
self.logs[job_id].close()
mkdir(os.path.dirname(filename))
self.logs[job_id] = FileHandler(filename)
else:
self.logger.info("[%s] Receiving logs from a new job", job_id)
mkdir(os.path.dirname(filename))
self.logs[job_id] = FileHandler(filename)
if message_lvl == "results":
try:
job = TestJob.objects.get(pk=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] Unknown job id", job_id)
return
meta_filename = create_metadata_store(message_msg, job, level)
ret = map_scanned_results(results=message_msg, job=job, meta_filename=meta_filename)
if not ret:
self.logger.warning(
"[%s] Unable to map scanned results: %s",
job_id, message)
# Mark the file handler as used
# TODO: try to use a more pythonnic way
self.logs[job_id].last_usage = time.time()
# n.b. logging here would produce a log entry for every message in every job.
# The format is a list of dictionaries
message = "- %s" % message
# Write data
f_handler = self.logs[job_id].fd
f_handler.write(message)
f_handler.write('\n')
f_handler.flush()
# TODO: keep the file handler to avoid calling open for each line
filename = os.path.join(options['output_dir'],
"job-%s" % job_id,
'output.yaml')
with open(filename, 'a+') as f_out:
f_out.write(message)
f_out.write('\n')
def logging_socket(self, options):
msg = self.pull_socket.recv_multipart()
try:
(job_id, level, name, message) = msg
except ValueError:
# do not let a bad message stop the master.
self.logger.error("Failed to parse log message, skipping: %s", msg)
return
try:
scanned = yaml.load(message)
except yaml.YAMLError:
self.logger.error("[%s] data are not valid YAML, dropping", job_id)
return
# Look for "results" level
try:
message_lvl = scanned["lvl"]
message_msg = scanned["msg"]
except KeyError:
self.logger.error(
"[%s] Invalid log line, missing \"lvl\" or \"msg\" keys: %s",
job_id, message)
return
if message_lvl == "results":
try:
job = TestJob.objects.get(pk=job_id)
except TestJob.DoesNotExist:
self.logger.error("[%s] Unknown job id", job_id)
return
ret = map_scanned_results(results=message_msg, job=job)
if not ret:
self.logger.warning("[%s] Unable to map scanned results: %s",
job_id, message)
# Clear filename
if '/' in level or '/' in name:
self.logger.error(
"[%s] Wrong level or name received, dropping the message",
job_id)
return
filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % (
options['output_dir'], job_id, level.split('.')[0], level, name)
# Find the handler (if available)
if job_id in self.logs:
if filename != self.logs[job_id].filename:
# Close the old file handler
self.logs[job_id].close()
mkdir(os.path.dirname(filename))
self.logs[job_id] = FileHandler(filename)
else:
self.logger.info("[%s] Receiving logs from a new job", job_id)
mkdir(os.path.dirname(filename))
self.logs[job_id] = FileHandler(filename)
# Mark the file handler as used
# TODO: try to use a more pythonnic way
self.logs[job_id].last_usage = time.time()
# n.b. logging here would produce a log entry for every message in every job.
# The format is a list of dictionaries
message = "- %s" % message
# Write data
f_handler = self.logs[job_id].fd
f_handler.write(message)
f_handler.write('\n')
f_handler.flush()
# TODO: keep the file handler to avoid calling open for each line
filename = os.path.join(options['output_dir'], "job-%s" % job_id,
'output.yaml')
with open(filename, 'a+') as f_out:
f_out.write(message)
f_out.write('\n')
