def run(self):
"""Execute the whole pipeline."""
def _shutdown_cluster(signum, frame):
logging.info("Shutdown urgently.")
Master.shutdown()
os._exit(0)
try:
signal.signal(signal.SIGINT, _shutdown_cluster)
signal.signal(signal.SIGTERM, _shutdown_cluster)
for step_name in PipelineConfig.steps:
step_cfg = UserConfig().data.get(step_name)
General.step_name = step_name
ClassFactory().set_current_step(step_cfg)
# load Config obj form desc
load_conf_from_desc(PipeStepConfig, step_cfg)
logger.info("Start pipeline step: [{}]".format(step_name))
PipeStep().do()
except Exception:
logger.error("Failed to run pipeline.")
logger.error(traceback.format_exc())
try:
Master.shutdown()
except Exception:
logger.error("Failed to shutdown dask cluster.")
logger.error(traceback.format_exc())
def run(self):
"""Execute the whole pipeline."""
def _shutdown_cluster(signum, frame):
logging.info("Shutdown urgently.")
Master.shutdown()
os._exit(0)
try:
signal.signal(signal.SIGINT, _shutdown_cluster)
signal.signal(signal.SIGTERM, _shutdown_cluster)
for step_name in self._steps:
step_cfg = self.cfg.get(step_name)
self.cfg.general["step_name"] = step_name
ClassFactory().set_current_step(step_cfg)
logger.info("Start pipeline step: [{}]".format(step_name))
PipeStep().do()
except Exception:
logger.error("Failed to run pipeline.")
logger.error(traceback.format_exc())
try:
Master.shutdown()
except Exception:
logger.error("Failed to shutdown dask cluster.")
logger.error(traceback.format_exc())
class SpNasPipeStep(PipeStep):
"""PipeStep is the base components class that can be added in Pipeline."""
def __init__(self):
"""Initialize SpNasPipeStep."""
super().__init__()
self.generator = Generator()
self.master = Master()
def do(self):
"""Do the main task in this pipe step."""
logger.info("SpNasPipeStep started")
while not self.generator.is_completed:
id, spnas_sample = self.generator.search_alg.search()
cls_trainer = ClassFactory.get_cls('trainer')
trainer = cls_trainer(spnas_sample=spnas_sample, id=id)
logging.info("submit trainer(id={})!".format(id))
self.master.run(trainer)
finished_trainer_info = self.master.pop_finished_worker()
logger.debug(finished_trainer_info)
self.update_generator(self.generator, finished_trainer_info)
self.master.join()
finished_trainer_info = self.master.pop_all_finished_train_worker()
self.update_generator(self.generator, finished_trainer_info)
def update_generator(self, generator, worker_info):
"""Get finished worker's info, and use it to update target `generator`.
Will get the finished worker's working dir, and then call the function
`generator.update(step_name, worker_id)`.
:param Generator generator: The target `generator` need to update.
:param worker_info: `worker_info` is the finished worker's info, usually
a dict or list of dict include `step_name` and `worker_id`.
:type worker_info: dict or list of dict.
"""
if worker_info is None:
return
if not isinstance(worker_info, list):
worker_info = [worker_info]
for one_info in worker_info:
step_name = one_info["step_name"]
worker_id = one_info["worker_id"]
logging.info("update generator, step name: {}, worker id: {}".format(step_name, worker_id))
try:
generator.search_alg.update({"step_name": step_name, "worker_id": worker_id})
except Exception:
logging.error("Failed to upgrade generator, step_name={}, worker_id={}.".format(step_name, worker_id))
logging.error(traceback.format_exc())
class SpNasPipeStep(NasPipeStep):
"""PipeStep is the base components class that can be added in Pipeline."""
def __init__(self):
super().__init__()
self.generator = Generator()
self.master = Master()
def do(self):
"""Do the main task in this pipe step."""
logger.info("SpNasPipeStep started...")
while not self.generator.is_completed:
id, sample = self.generator.search_alg.search()
cls_trainer = ClassFactory.get_cls('trainer')
trainer = cls_trainer(sample, id)
logging.info("submit trainer(id={})!".format(id))
self.master.run(trainer)
finished_trainer_info = self.master.pop_finished_worker()
print(finished_trainer_info)
self.update_generator(self.generator, finished_trainer_info)
self.master.join()
finished_trainer_info = self.master.pop_all_finished_train_worker()
self.update_generator(self.generator, finished_trainer_info)
def __init__(self):
"""Initialize SpNasPipeStep."""
super().__init__()
self.generator = Generator()
self.master = Master()
def __init__(self):
super().__init__()
self.generator = Generator()
self.master = Master()
def _shutdown_cluster(signum, frame):
logging.info("Shutdown urgently.")
Master.shutdown()
os._exit(0)