def print_current_state(channel: WorkerJobPool):
print("Known workers")
w_format = "{:45s}{:30s}"
print(w_format.format("Service id", "Current state"))
print("-" * 80)
for worker in channel.list_known_workers():
print(w_format.format(worker.service_id, worker.state))
print("\nKnown jobs")
j_format = "{:26s}{:30s}{:30s}"
print(j_format.format("Service id", "Job id", "Current state"))
print("-" * 80)
for job in channel.list_known_jobs():
print(j_format.format(job.service_id, job.job_id, job.state))
if job.file_name is not None:
print(f" File name: {job.file_name}")
if len(job.message) > 0:
print(f" Message: {job.message}")
print("\nKnown commands")
c_format = j_format
print(c_format.format("Job id", "Command id", "Current state"))
print("-" * 80)
for command in channel.list_known_commands():
print(
c_format.format(command.job_id, command.command_id, command.state))
if len(command.message) > 0:
print(" Message: {}".format(command.message))
def prepare_write_job(args: argparse.Namespace) -> WriteJob:
global JOB_HANDLER
global ACK_TIMEOUT
file_name = args.filename
job_id = args.job_id
host = args.broker
command_topic = args.command_status_topic
pool_topic = args.job_pool_topic
config = args.config
ACK_TIMEOUT = args.timeout
command_channel = WorkerJobPool(f"{host}/{pool_topic}",
f"{host}/{command_topic}")
JOB_HANDLER = JobHandler(worker_finder=command_channel)
with open(config, "r") as f:
nexus_structure = f.read()
if job_id:
write_job = WriteJob(nexus_structure,
file_name,
host,
datetime.now(),
job_id=job_id)
else:
write_job = WriteJob(
nexus_structure,
file_name,
host,
datetime.now(),
)
return write_job
def create_job_handler(args: argparse.Namespace, job_id: str) -> JobHandler:
host = args.broker
command_topic = args.command_status_topic
pool_topic = args.job_pool_topic
command_channel = WorkerJobPool(f"{host}/{pool_topic}",
f"{host}/{command_topic}")
job_handler = JobHandler(worker_finder=command_channel, job_id=job_id)
# Required for formation of the handler.
time.sleep(3)
return job_handler
import time
from datetime import datetime, timedelta
from file_writer_control import CommandState
from file_writer_control import JobHandler
from file_writer_control import JobState
from file_writer_control import WorkerJobPool
from file_writer_control import WriteJob
if __name__ == "__main__":
kafka_host = "dmsc-kafka01:9092"
worker_job_pool = WorkerJobPool("{}/job_topic".format(kafka_host),
"{}/command_topic".format(kafka_host))
job_handler = JobHandler(worker_finder=worker_job_pool)
start_time = datetime.now()
with open("file_writer_config.json", "r") as f:
nexus_structure = f.read()
write_job = WriteJob(
nexus_structure,
"{0:%Y}-{0:%m}-{0:%d}_{0:%H}{0:%M}.nxs".format(start_time),
kafka_host,
start_time,
)
print("Starting write job")
start_handler = job_handler.start_job(write_job)
while not start_handler.is_done():
print("Waiting: {}".format(start_handler.get_state()))
if start_handler.get_state() == CommandState.ERROR:
print("Got error when starting job. The error was: {}".format(
start_handler.get_message()))
import time
from datetime import datetime, timedelta
from file_writer_control import JobHandler
from file_writer_control import WorkerJobPool
from file_writer_control import WriteJob
from os import path
if __name__ == '__main__':
kafka_host = 'localhost:9092'
worker_job_pool = WorkerJobPool(f'{kafka_host}/Loki_jobPool',
f'{kafka_host}/Loki_writerCommand')
job_handler = JobHandler(worker_finder=worker_job_pool)
nexus_structure_file = path.join('..', 'NXtomo.json')
start_time = datetime.now()
with open(nexus_structure_file, 'r') as f:
nexus_structure = f.read()
write_job = WriteJob(
nexus_structure,
"{0:%Y}-{0:%m}-{0:%d}_{0:%H}{0:%M}.nxs".format(start_time),
kafka_host,
start_time,
)
print('Starting write job')
start_handler = job_handler.start_job(write_job)
while not start_handler.is_done():
time.sleep(1)
stop_time = start_time + timedelta(seconds=30)
stop_handler = job_handler.set_stop_time(stop_time)
while not stop_handler.is_done():
kafka_host = "dmsc-kafka01:9092"
# Job pool is where idle file-writers listen for start messages.
# The start message can come from any instrument and only one file-writer will
# respond to it.
job_pool_topic = "job_pool_topic"
# Control topic is a "private" topic dedicated to a specific instrument.
# This is where the selected file-writer writes status messages and
# listens for stop messages from that instrument.
control_topic = "ymir_control_topic"
with open("file_writer_config.json", "r") as f:
nexus_structure = f.read()
command_channel = WorkerJobPool(f"{kafka_host}/{job_pool_topic}",
f"{kafka_host}/{control_topic}")
job_handler = JobHandler(worker_finder=command_channel)
start_time = datetime.now()
write_job = WriteJob(
nexus_structure,
"{0:%Y}-{0:%m}-{0:%d}_{0:%H}{0:%M}.nxs".format(start_time),
kafka_host,
start_time,
control_topic=control_topic,
)
print("Starting write job")
start_handler = job_handler.start_job(write_job)
while not start_handler.is_done():
time.sleep(1)
print(w_format.format(worker.service_id, worker.state))
print("\nKnown jobs")
j_format = "{:26s}{:30s}{:30s}"
print(j_format.format("Service id", "Job id", "Current state"))
print("-" * 80)
for job in channel.list_known_jobs():
print(j_format.format(job.service_id, job.job_id, job.state))
if job.file_name is not None:
print(f" File name: {job.file_name}")
if len(job.message) > 0:
print(f" Message: {job.message}")
print("\nKnown commands")
c_format = j_format
print(c_format.format("Job id", "Command id", "Current state"))
print("-" * 80)
for command in channel.list_known_commands():
print(
c_format.format(command.job_id, command.command_id, command.state))
if len(command.message) > 0:
print(" Message: {}".format(command.message))
if __name__ == "__main__":
kafka_host = "dmsc-kafka01:9092"
worker_job_pool = WorkerJobPool(f"{kafka_host}/job_pool_topic",
f"{kafka_host}/command_topic")
time.sleep(10)
print_current_state(worker_job_pool)