def __init__(self, project_dir=None, max_training_processes=1, response_log=None, emulation_mode=None, remote_storage=None, component_builder=None, model_server=None, wait_time_between_pulls=None): self._training_processes = max(max_training_processes, 1) self._current_training_processes = 0 self.responses = self._create_query_logger(response_log) self.project_dir = config.make_path_absolute(project_dir) self.emulator = self._create_emulator(emulation_mode) self.remote_storage = remote_storage self.model_server = model_server self.wait_time_between_pulls = wait_time_between_pulls if component_builder: self.component_builder = component_builder else: self.component_builder = ComponentBuilder(use_cache=True) self.project_store = self._create_project_store(project_dir) # tensorflow sessions are not fork-safe, # and training processes have to be spawned instead of forked. See # https://github.com/tensorflow/tensorflow/issues/5448#issuecomment # -258934405 multiprocessing.set_start_method('spawn', force=True) self.pool = ProcessPool(self._training_processes)
def __init__(self, project_dir=None, max_training_processes=1, response_log=None, emulation_mode=None, remote_storage=None, component_builder=None, model_server=None, wait_time_between_pulls=None): self._training_processes = max(max_training_processes, 1) self._current_training_processes = 0 self.responses = self._create_query_logger(response_log) self.project_dir = config.make_path_absolute(project_dir) self.emulator = self._create_emulator(emulation_mode) self.remote_storage = remote_storage self.model_server = model_server self.wait_time_between_pulls = wait_time_between_pulls if component_builder: self.component_builder = component_builder else: self.component_builder = ComponentBuilder(use_cache=True) self.project_store = self._create_project_store(project_dir) self.pool = ProcessPool(self._training_processes)
def __init__(self, project_dir=None, max_training_processes=1, response_log=None, emulation_mode=None, remote_storage=None, component_builder=None): self._training_processes = max(max_training_processes, 1) self.responses = self._create_query_logger(response_log) self.project_dir = config.make_path_absolute(project_dir) self.emulator = self._create_emulator(emulation_mode) self.remote_storage = remote_storage if component_builder: self.component_builder = component_builder else: self.component_builder = ComponentBuilder(use_cache=True) self.project_manager = ProjectManager.create(project_dir, remote_storage, component_builder) self.project_store = self.project_manager.get_projects() self.pool = ProcessPool(self._training_processes)
def __init__(self, config, component_builder): self._training_processes = config['max_training_processes'] if config['max_training_processes'] > 0 else 1 self.config = config self.responses = self._create_query_logger(config) self.model_dir = config['path'] self.emulator = self._create_emulator() self.component_builder = component_builder if component_builder else ComponentBuilder(use_cache=True) self.project_store = self._create_project_store() self.pool = ProcessPool(self._training_processes)
def __init__(self, config, component_builder): self._training_processes = config['max_training_processes'] if config['max_training_processes'] > 0 else 1 self.config = config self.responses = DataRouter._create_query_logger(config['response_log']) self._trainings_queued = 0 self.model_dir = config['path'] self.token = config['token'] self.emulator = self.__create_emulator() self.component_builder = component_builder if component_builder else ComponentBuilder(use_cache=True) self.model_store = self.__create_model_store() self.pool = ProcessPool(self._training_processes)
def __init__( self, project_dir=None, max_training_processes=1, response_log=None, emulation_mode=None, remote_storage=None, component_builder=None, model_server=None, wait_time_between_pulls=None, ): self._training_processes = max(max_training_processes, 1) self._current_training_processes = 0 self.responses = self._create_query_logger(response_log) self.project_dir = config.make_path_absolute(project_dir) self.emulator = self._create_emulator(emulation_mode) self.remote_storage = remote_storage self.model_server = model_server self.wait_time_between_pulls = wait_time_between_pulls if component_builder: self.component_builder = component_builder else: self.component_builder = ComponentBuilder(use_cache=True) # TODO: Should be moved to separate method loop = asyncio.get_event_loop() if loop.is_closed(): loop = asyncio.new_event_loop() self.project_store = loop.run_until_complete( self._create_project_store(self.project_dir) ) loop.close() # tensorflow sessions are not fork-safe, # and training processes have to be spawned instead of forked. See # https://github.com/tensorflow/tensorflow/issues/5448#issuecomment # -258934405 multiprocessing.set_start_method("spawn", force=True) self.pool = ProcessPool(self._training_processes)