def __init__(self, args, id): self.args = args self.id = id #### Rollout Actor is a template used for MP ##### self.manager = Manager() self.rollout_actor = self.manager.list() for _ in range(args.config.num_agents): if args.ps == 'trunk': self.rollout_actor.append( MultiHeadActor(args.state_dim, args.action_dim, args.hidden_size, args.config.num_agents)) else: if args.algo_name == 'TD3': self.rollout_actor.append( Actor(args.state_dim, args.action_dim, args.hidden_size, policy_type='DeterministicPolicy')) else: self.rollout_actor.append( Actor(args.state_dim, args.action_dim, args.hidden_size, policy_type='GaussianPolicy')) if self.args.ps == 'full' or self.args.ps == 'trunk': break #Only need one for homogeneous workloads
def __init__(self, model_shared_dict, buffer_queue, device, batch_size=16, max_workers=140, episode_length=1000): super(ModelInferenceServer, self).__init__() self.model_shared_dict = model_shared_dict self.workers = [] self.pipes = [] # Communication channels with workers self.backlog = Queue() self.return_dict = Manager().dict() self.buffer_queue = buffer_queue self.model = None self.device = device self.batch_size = batch_size self.max_workers = max_workers # 140 seems to be the optimum self.episode_length = episode_length self.shutdown = False self.empty_count = 0 self.avg_queue_size = 0 self.avg_batch_size = 0 self.avg_total_time = 0 self.avg_batch_pull_time = 0 self.avg_batch_infr_time = 0 self.avg_batch_retu_time = 0
def run_hpo(TrainerClass, global_config: Config): # TODO: Is it ok to assume that we always have more CPUs than concurrent experiments? configs = spawn_configs_for_hpo(global_config) clean_dir(os.path.join(global_config.experiment_dir, global_config.exp_name, 'summaries'), create=True) n_parallel = global_config.hpo.get('num_parallel_experiments_per_gpu', 1) \ * (len(global_config.firelab.gpus) \ // global_config.hpo.get('num_gpus_per_experiment', 1)) logger.info(f'Total number of experiments to run: {len(configs)}') logger.info(f'Num concurrent HPO experiments to run: {n_parallel}') gpus: Tuple[int] = tuple(global_config.firelab.gpus) gpus_usage = Manager().list([0] * len(gpus)) gpus_usage_lock = Manager().Lock() futures = [] with ProcessPoolExecutor(n_parallel) as executor: for config in configs: args = [ TrainerClass, config, global_config.hpo.get('num_gpus_per_experiment', 1), global_config.hpo.get('num_parallel_experiments_per_gpu', 1), gpus_usage, gpus_usage_lock, gpus ] future = executor.submit(run_single_hpo_experiment, *args) futures.append(future) wait(futures)
def colorize(request, context): try: manager = Manager() return_dict = manager.dict() p = Process(target=mp_colorize, args=(request.img_input, request.render_factor, return_dict)) p.start() p.join() response = return_dict.get("response", None) if not response or "error" in response: error_msg = response.get("error", None) if response else None log.error(error_msg) context.set_details(error_msg) context.set_code(grpc.StatusCode.INTERNAL) return Output() log.debug("colorize({})={}".format(request.img_input[:50], response["img_colorized"][:50])) return Output(img_colorized=response["img_colorized"]) except Exception as e: traceback.print_exc() log.error(e) return Output()
def __init__(self, config, share_batches=True, manager=None, new_process=True): if new_process == True and manager is None: manager = Manager() self.knows = Semaphore(0) # > 0 if we know if any are coming # == 0 if DatasetReader is processing a command self.working = Semaphore(1 if new_process else 100) self.finished_reading = Lock( ) # locked if we're still reading from file # number of molecules that have been sent to the pipe: self.in_pipe = Value('i', 0) # Tracking what's already been sent through the pipe: self._example_number = Value('i', 0) # The final kill switch: self._close = Value('i', 0) self.command_queue = manager.Queue(10) self.molecule_pipeline = None self.batch_queue = Queue(config.data.batch_queue_cap ) #manager.Queue(config.data.batch_queue_cap) self.share_batches = share_batches self.dataset_reader = DatasetReader("dataset_reader", self, config, new_process=new_process) if new_process: self.dataset_reader.start()
def async_build_examples( self, data_type: str, dials: List[Tuple[str, dict]]) -> Tuple[list, list]: """Use multiprocessing to process raw dialogue data. Args: data_type: train, dev or test dials: raw dialogues data Returns: new examples by all processes """ neg_examples = Manager().list() pos_examples = Manager().list() dials4single_process = (len(dials) - 1) // self.config['num_processes'] + 1 print(f'Single process have {dials4single_process} dials ...') pool = Pool(self.config['num_processes']) for i in range(self.config['num_processes']): pool.apply_async(func=self.iter_dials, args=(dials[dials4single_process * i:dials4single_process * (i + 1)], data_type, pos_examples, neg_examples, i)) pool.close() pool.join() pos_examples = list(pos_examples) neg_examples = list(neg_examples) return neg_examples, pos_examples
def run(self, nepoch, batchsize=None, loss='variance', ndist=1): if ndist == 1: self.distributed_training = False self._worker(nepoch, batchsize, loss) else: self.distributed_training = True processes = [] manager = Manager() obs_data = manager.list() for rank in range(ndist): p = Process(target=self.init_process, args=(obs_data, rank, ndist, nepoch, batchsize, loss)) p.start() processes.append(p) for p in processes: p.join() self.obs_dict = obs_data
def __init__(self, capacity, buffer_gpu, filter_c=None): self.capacity = capacity self.buffer_gpu = buffer_gpu self.filter_c = filter_c self.manager = Manager() self.tuples = self.manager.list( ) #Temporary shared buffer to get experiences from processes self.s = [] self.ns = [] self.a = [] self.r = [] self.done = [] self.global_reward = [] # Temporary tensors that cane be loaded in GPU for fast sampling during gradient updates (updated each gen) --> Faster sampling - no need to cycle experiences in and out of gpu 1000 times self.sT = None self.nsT = None self.aT = None self.rT = None self.doneT = None self.global_rewardT = None self.pg_frames = 0 self.total_frames = 0 #Priority indices self.top_r = None self.top_g = None #Stats self.rstats = {'min': None, 'max': None, 'mean': None, 'std': None} self.gstats = {'min': None, 'max': None, 'mean': None, 'std': None}
def per_step(valLoader, model, criterion, downsamplingFactor): model.eval() criterion.eval() avgPER = 0 varPER = 0 nItems = 0 print("Starting the PER computation through beam search") bar = progressbar.ProgressBar(maxval=len(valLoader)) bar.start() for index, data in enumerate(valLoader): bar.update(index) with torch.no_grad(): seq, sizeSeq, phone, sizePhone = prepare_data(data) c_feature = model(seq) sizeSeq = sizeSeq / downsamplingFactor predictions = torch.nn.functional.softmax(criterion.getPrediction(c_feature), dim=2).cpu() c_feature = c_feature phone = phone.cpu() sizeSeq = sizeSeq.cpu() sizePhone = sizePhone.cpu() mutex = Lock() manager = Manager() poolData = manager.list() processes = [] for b in range(sizeSeq.size(0)): l_ = min(sizeSeq[b] // 4, predictions.size(1)) s_ = sizePhone[b] p = torch.multiprocessing.Process(target=get_local_per, args=(poolData, mutex, predictions[b, :l_].view(l_, -1).numpy(), phone[b, :s_].view(-1).numpy().astype(np.int32), criterion.BLANK_LABEL)) p.start() processes.append(p) for p in processes: p.join() avgPER += sum([x for x in poolData]) varPER += sum([x*x for x in poolData]) nItems += len(poolData) bar.finish() avgPER /= nItems varPER /= nItems varPER -= avgPER**2 print(f"Average PER {avgPER}") print(f"Standard deviation PER {math.sqrt(varPER)}")
def __init__(self, args, id): self.args = args self.id = id #### Rollout Actor is a template used for MP ##### self.manager = Manager() self.rollout_actor = self.manager.list() self.rollout_actor.append(MultiHeadActor(args.state_dim, args.action_dim, args.hidden_size, args.config.num_agents))
def __init__(self, args): self.args = args self.agents = [pg.TD3(args) for _ in range(self.args.num_rover)] #Load to GPU for ag in self.agents: ag.to_cuda() ###### Buffer is agent's own data self generated via its rollouts ######### self.buffers = [Buffer() for _ in range(self.args.num_rover)] self.noise_gen = OU_handle.get_list_generators(NUM_WORKERS, args.action_dim) ######### Multiprocessing TOOLS ######### self.manager = Manager() self.data_bucket = [ self.manager.list() for _ in range(args.num_rover) ] #Experience list stores experiences from all processes ######### TRAIN ROLLOUTS WITH ACTION NOISE ############ self.models_bucket = self.manager.list() model_template = models.Actor(args) for _ in range(self.args.num_rover): self.models_bucket.append(models.Actor(args)) self.task_pipes = [Pipe() for _ in range(NUM_WORKERS)] self.result_pipes = [Pipe() for _ in range(NUM_WORKERS)] self.train_workers = [ Process(target=rollout_worker, args=(self.args, i, self.task_pipes[i][1], self.result_pipes[i][0], self.noise_gen[i], self.data_bucket, self.models_bucket, model_template)) for i in range(NUM_WORKERS) ] for worker in self.train_workers: worker.start() ######## TEST ROLLOUT POLICY ############ self.test_task_pipe = Pipe() self.test_result_pipe = Pipe() self.test_worker = Process(target=rollout_worker, args=(self.args, 0, self.test_task_pipe[1], self.test_result_pipe[0], None, self.data_bucket, self.models_bucket, model_template)) self.test_worker.start() #### STATS AND TRACKING WHICH ROLLOUT IS DONE ###### self.best_policy = models.Actor(args) #Best policy found by PF yet self.best_score = -999 self.test_score = None self.test_eval_flag = True self.rollout_scores = [None for _ in range(NUM_WORKERS)] self.best_rollout_score = -999 self.train_eval_flag = [True for _ in range(NUM_WORKERS)] self.update_budget = 0
def __init__(self, discount_factor): super().__init__() self.frames = [] self.value = [] self.start = 0 self.discount_factor = discount_factor self.game_length = 0 self.gl = [] self.collected_rollouts = 0 self.reward_total = 0 self.rollouts = Manager().Queue()
def __init__(self, capacity, buffer_gpu=False): self.capacity = capacity self.buffer_gpu = buffer_gpu self.counter = 0 self.manager = Manager() self.tuples = self.manager.list( ) #Temporary shared buffer to get experiences from processes self.s = [] self.ns = [] self.a = [] self.r = [] self.done = []
def hdf5_to_dict(hdf5_file_path): data = {} manager = Manager() md_data = manager.dict() with h5py.File(hdf5_file_path, 'r') as read_file: for scene in tqdm(read_file.keys()): # data[scene] = {} data[scene] = manager.dict() for pos in read_file[scene].keys(): data[scene][pos] = read_file[scene][pos][()] md_data.update(data) return md_data
def run_tracker(self): """ Run self.pipeline on Ali vos """ num_gpu = self._hyper_params["device_num"] all_devs = [torch.device("cuda:%d" % i) for i in range(num_gpu)] logger.info('runing test on devices {}'.format(all_devs)) ALI_root = self._hyper_params["data_root"] logger.info('Using dataset %s at: %s' % (self.dataset_name, ALI_root)) # setup dataset dataset = davis_benchmark.load_dataset( ALI_root, self.dataset_name ) # OrderedDict:30 bike-packing, blackswan, bmx-trees... self.dataset = dataset keys = list(dataset.keys()) # list:30 keys.sort() nr_records = len(keys) pbar = tqdm(total=nr_records) mean_speed = -1 speed_list = [] manager = Manager() speed_queue = manager.Queue(500) # set worker if num_gpu == 0: self.worker(keys, all_devs[0], self.dataset, speed_queue) for i in range(nr_records): s = speed_queue.get() speed_list.append(s) pbar.update(1) else: nr_video = math.ceil(nr_records / num_gpu) # 每个gpu负载的测试视频数量 procs = [] for i in range(num_gpu): start = i * nr_video # 0 end = min(start + nr_video, nr_records) # 两块gpu, 30/2 = 15 split_records = keys[start:end] proc = mp.Process(target=self.worker, args=(split_records, all_devs[i], self.dataset, speed_queue)) logger.info('process:%d, start:%d, end:%d' % (i, start, end)) proc.start() procs.append(proc) for i in range(nr_records): s = speed_queue.get() speed_list.append(s) pbar.update(1) for p in procs: p.join() # print result mean_speed = float(np.mean(speed_list)) logger.info('Mean Speed: {:.2f} FPS'.format(mean_speed)) self._state['speed'] = mean_speed
def __init__(self, args): self.args = args ctx = get_context('spawn') self.manager = Manager() self.smart_value = self.manager.Value('i', 0) self.connection = Pipe() self.queue = self.manager.Queue() # self.stream_process = ctx.Process(target=client, args=(self.args, None)) # self.detection_process = ctx.Process(target=server, args=(None, None)) self.input_thread = threading.Thread(name='input_thread', target=self.collect_rowdata) self.detection_thread = threading.Thread(name='input_thread', target=self.detection_loop)
def run(self): m = Manager() global_ep, res_dict = m.dict(), m.dict() global_ep['g_ep'] = 0 global_ep['g_sum_step'] = 0 res_dict['res_q'] = [] rew_queue = m.Queue() agent_input_dict_list = m.list([self.input_dict for _ in range(1)]) agent_return_dict_list = m.list([m.dict() for _ in range(1)]) a3c_workers = [ A3CWorker(global_ep, res_dict, rew_queue, agent_input_dict_list[i], agent_return_dict_list[i], i) for i in range(1) ] [w.start() for w in a3c_workers] [w.join() for w in a3c_workers] res = res_dict['res_q'] print('game_result:', res) for agent_return_dict in agent_return_dict_list: print(agent_return_dict) win_rate, tie_rate, lose_rate, step_game = self.calculate_statistics( agent_return_dict_list) print(win_rate, tie_rate, lose_rate, step_game) self.return_dict[int(self.nth_pbt_process)] = [win_rate, step_game]
def __init__(self, args, model_constructor, env_constructor): self.args = args #MP TOOLS self.manager = Manager() #Algo sac_keyargs = {} sac_keyargs['autotune'] = args.autotune sac_keyargs['entropy'] = True self.algo = SAC(args, model_constructor, args.gamma, **sac_keyargs) # #Save best policy # self.best_policy = model_constructor.make_model('actor') #Init BUFFER self.replay_buffer = Buffer(args.buffer_size) self.data_bucket = self.replay_buffer.tuples #Initialize Rollout Bucket self.rollout_bucket = self.manager.list() self.rollout_bucket.append(model_constructor.make_model('Gaussian_FF')) ############## MULTIPROCESSING TOOLS ################### #Learner rollout workers self.task_pipes = [Pipe() for _ in range(args.rollout_size)] self.result_pipes = [Pipe() for _ in range(args.rollout_size)] self.workers = [Process(target=rollout_worker, args=(id, 'pg', self.task_pipes[id][1], self.result_pipes[id][0], self.data_bucket, self.rollout_bucket, env_constructor)) for id in range(args.rollout_size)] for worker in self.workers: worker.start() self.roll_flag = [True for _ in range(args.rollout_size)] #Test bucket self.test_bucket = self.manager.list() self.test_bucket.append(model_constructor.make_model('Gaussian_FF')) #5 Test workers self.test_task_pipes = [Pipe() for _ in range(env_constructor.dummy_env.test_size)] self.test_result_pipes = [Pipe() for _ in range(env_constructor.dummy_env.test_size)] self.test_workers = [Process(target=rollout_worker, args=(id, 'test', self.test_task_pipes[id][1], self.test_result_pipes[id][0], None, self.test_bucket, env_constructor)) for id in range(env_constructor.dummy_env.test_size)] for worker in self.test_workers: worker.start() self.test_flag = False #Trackers self.best_score = 0.0; self.gen_frames = 0; self.total_frames = 0; self.test_score = None; self.test_std = None; self.test_trace = []; self.rollout_fits_trace = [] self.ep_len = 0 self.r1_reward = 0 self.num_footsteps = 0 self.best_shaped_score = 0.0
def eval_init(self): self.processes = [] M = Manager() self.res_Q = M.Queue() self.sync_Q = [] for idx in range(self.process_num): print('process',idx) syncq = M.Queue() p = EvalProcess(self.sub_envs_args[idx],self.global_model, self.res_Q, syncq, idx) self.sync_Q.append(syncq) self.processes.append(p) for p in self.processes: p.start()
class TestAgent: """Learner object encapsulating a local learner Parameters: algo_name (str): Algorithm Identifier state_dim (int): State size action_dim (int): Action size actor_lr (float): Actor learning rate critic_lr (float): Critic learning rate gamma (float): DIscount rate tau (float): Target network sync generate init_w (bool): Use kaimling normal to initialize? **td3args (**kwargs): arguments for TD3 algo """ def __init__(self, args, id): self.args = args self.id = id #### Rollout Actor is a template used for MP ##### self.manager = Manager() self.rollout_actor = self.manager.list() self.rollout_actor.append(MultiHeadActor(args.state_dim, args.action_dim, args.hidden_size, args.config.num_agents)) def make_champ_team(self, agents): for agent_id, agent in enumerate(agents): if self.args.popn_size <= 1: #Testing without Evo agent.update_rollout_actor() mod.hard_update(self.rollout_actor[agent_id], agent.rollout_actor[0]) else: mod.hard_update(self.rollout_actor[agent_id], agent.popn[agent.champ_ind])
def __init__(self, chk_dir, chk, keep_epoch_chk=True, overwrite=True, mode=CFMode.AUTO, chk_prefix='model_v_'): self.logger = logging.getLogger(__name__) self.chk_dir = chk_dir self.chk = chk self.keep_epoch_chk = keep_epoch_chk self.overwrite = overwrite self.chk_prefix = chk_prefix self.mode = mode self.chk_epoch_subdir = 'epoch' self.mp_manager = Manager() self.snapshot_copy = None self.cpu_side = False # Active snapshot, if true, don't snapshot again self.active_snapshot = Value('i', 0) self.lock = Lock() self.in_progress_snapshot = Value('i', 0) # Handle to the process performing checkpoint # Can be only one at any instant. A new checkpoint # cannot start unless the previous one completes self.chk_process = None # `overwrite` supersedes if False if self.overwrite is False and self.keep_epoch_chk is False: self.keep_epoch_chk = True # Global ID of checkpoints being written # Used to format the checkpoint path # Instantiate from chk when restoring self.chk_global_id = -1 # Sorted List of available checkpoints (fnames) self.available_chk_iters = self.mp_manager.list() self.available_chk_epochs = self.mp_manager.list() self.initalize_chk_dir() self.logger.info("Available checkpoints : ") for item in self.available_chk_iters: self.logger.info(item)
def __call__(self, instances: Iterable[Instance], num_epochs: int = None, shuffle: bool = True) -> Iterator[TensorDict]: # If you run it forever, the multiprocesses won't shut down correctly. # TODO(joelgrus) find a solution for this if num_epochs is None: raise ConfigurationError( "Multiprocess Iterator must be run for a fixed number of epochs" ) manager = Manager() output_queue = manager.Queue(self.output_queue_size) input_queue = manager.Queue(self.output_queue_size * self.batch_size) # Start process that populates the queue. self.queuer = Process(target=_queuer, args=(instances, input_queue, self.num_workers, num_epochs)) self.queuer.start() # Start the tensor-dict workers. for i in range(self.num_workers): args = (input_queue, output_queue, self.iterator, shuffle, i) process = Process(target=_create_tensor_dicts, args=args) process.start() self.processes.append(process) num_finished = 0 while num_finished < self.num_workers: item = output_queue.get() if isinstance(item, int): num_finished += 1 logger.info( f"worker {item} finished ({num_finished} / {self.num_workers})" ) else: yield item for process in self.processes: process.join() self.processes.clear() if self.queuer is not None: self.queuer.join() self.queuer = None
def run(self): # mp.set_start_method('spawn', force=True) with Manager() as manager: d = manager.dict() self.d = d d['train_progress'] = [] d['best_epoch'] = None d['best_epoch_summary'] = None d['model'] = None d['labels'] = None pqueue = mp.Queue() out_pqueue = mp.Queue() model = self.trainer_kwargs['model'] # model = copy.deepcopy(model) model.share_memory() self.trainer_kwargs['model'] = model self.trainer_kwargs['num_workers'] = 0 p = Process(target=main_q, args=(pqueue, out_pqueue, d)) p.daemon = True p.start() # pool.apply_async(main_q, args=(pqueue, out_pqueue, d, )) # pool.apply_async(main_train, args=(d, self.num_epochs, self.trainer_args, self.trainer_kwargs, self.datbaase_items)) # pool.starmap(main_q, [(pqueue, out_pqueue, d),]) pqueue.put(None) pqueue.put(self.num_epochs) pqueue.put(self.trainer_args) pqueue.put(self.trainer_kwargs) pqueue.put(self.database_items) p.join() # pool.close() # pool.join() print('Process results: ', len(d.keys())) # best_epoch = d['best_epoch'] # best_epoch_sumamry = d['best_epoch_summary'] # model = d['model'] # labels = d['labels'] self.d = get_queue_dict(out_pqueue, item_names=[ 'best_epoch', 'best_epoch_summary', 'model', 'labels', ]) best_epoch = self.d['best_epoch'] best_epoch_sumamry = self.d['best_epoch_summary'] model = model.load_state_dict(self.d['model']) labels = self.d['labels'] self.d = { "train_progress": d['train_progress'], } # best_epoch, best_epoch_summary = self.trainer.train(epochs=self.num_epochs) self.complete_func( self.host, { "best_epoch": best_epoch, "best_epoch_summary": best_epoch_sumamry, "model": model, "labels": labels, })
def __init__(self, args): self.args = args self.evolver = SSNE(self.args) #MP TOOLS self.manager = Manager() #Init population self.pop = self.manager.list() for _ in range(args.pop_size): self.pop.append(Actor(args)) #self.pop[-1].apply(utils.init_weights) self.best_policy = Actor(args) #Turn off gradients and put in eval mode for actor in self.pop: actor = actor.cpu() actor.eval() if SEED_POP: self.load_seed(args.model_save, self.pop) #Init BUFFER self.replay_buffer = Buffer(100000, self.args.data_folder) #MP TOOLS self.exp_list = self.manager.list() self.evo_task_pipes = [Pipe() for _ in range(args.pop_size)] self.evo_result_pipes = [Pipe() for _ in range(args.pop_size)] self.evo_workers = [ Process(target=rollout_worker, args=(i, self.evo_task_pipes[i][1], self.evo_result_pipes[i][1], None, self.exp_list, self.pop, DIFFICULTY, USE_RS, True, USE_SYNTHETIC_TARGET, XBIAS, ZBIAS, PHASE_LEN, None, EP_LEN, JGS)) for i in range(args.pop_size) ] for worker in self.evo_workers: worker.start() #Trackers self.buffer_added = 0 self.best_score = 0.0 self.frames_seen = 0.0 self.best_shaped_score = None self.eval_flag = [True for _ in range(args.pop_size)]
def __init__(self, dataset: Dataset, init_caching=False): self.dataset = dataset if init_caching: self.cache = dict() else: self.cache = Manager().dict() if init_caching: for idx, data in enumerate(tqdm(self.dataset)): self.cache[idx] = data
def __init__(self, capacity, buffer_gpu): self.capacity = capacity self.buffer_gpu = buffer_gpu self.counter = 0 self.manager = Manager() self.tuples = self.manager.list( ) #Temporary shared buffer to get experiences from processes self.s = [] self.ns = [] self.a = [] self.r = [] self.done = [] # Temporary tensors that cane be loaded in GPU for fast sampling during gradient updates (updated each gen) --> Faster sampling - no need to cycle experiences in and out of gpu 1000 times self.sT = None self.nsT = None self.aT = None self.rT = None self.doneT = None
def run(args): # setup N, D = args.N, args.D n_examples = args.n_train_examples n_threads = args.n_threads n_examples_per_thread = n_examples // n_threads # create policy network policy_network = alphatsp.util.get_policy_network(args.policy_network) # generate examples print("Generating examples and training...") manager = Manager() train_queue = manager.Queue() shared_dict = manager.dict() shared_dict["success"] = False producers = [] for _ in range(n_threads): producers.append( Process(target=generate_examples, args=(n_examples_per_thread, train_queue, args))) for p in producers: p.start() c = Process(target=train, args=(policy_network, train_queue, shared_dict, args)) c.start() for p in producers: p.join() train_queue.put(None) c.join() status = shared_dict["success"] if not status: print("Experiment failed.") return -1
def init_data(self): self.is_working = False self.semaphore = True self.is_change_bar = Value( c_bool, False) #whether user has dragged the slider,default: False self.frame_index = Value('i', 0) self.share_lock = Lock() #shared lock for frame_index self.share_lock2 = Lock() # shared lock for frame_index self.mutex = threading.Lock() self.timer = QTimer(self) # used for the updating of progress bar self.temp_timer = QTimer( self) #used for detecting whether the frame_total is given. self.frame_total = Value('i', -1) self.playable = Value(c_bool, True) self.is_working = Value(c_bool, False) manager = Manager() self.play_src = manager.Value(c_char_p, '0') #用于记录播放的视频地址 self.mode = None # 'online' or 'offline'
def __init_sample(self): if args.experience_replay is not '' and os.path.exists( args.experience_replay): self.D = torch.load(args.experience_replay) self.metrics['steps'], self.metrics['episodes'] = [ self.D.steps ] * self.D.episodes, list(range(1, self.D.episodes + 1)) elif not args.test: self.D = ExperienceReplay(args.experience_size, args.symbolic_env, self.env.observation_size, self.env.action_size, args.bit_depth, args.device) # Initialise dataset D with S random seed episodes print( "Start Multi Sample Processing -------------------------------" ) start_time = time.time() data_lists = [ Manager().list() for i in range(1, args.seed_episodes + 1) ] # Set Global Lists pipes = [Pipe() for i in range(1, args.seed_episodes + 1) ] # Set Multi Pipe workers_init_sample = [ Worker_init_Sample(child_conn=child, id=i + 1) for i, [parent, child] in enumerate(pipes) ] for i, w in enumerate(workers_init_sample): w.start() # Start Single Process pipes[i][0].send( data_lists[i]) # Parent_pipe send data using i'th pipes [w.join() for w in workers_init_sample] # wait sub_process done for i, [parent, child] in enumerate(pipes): # datas = parent.recv() for data in list(parent.recv()): if isinstance(data, tuple): assert len(data) == 4 self.D.append(data[0], data[1], data[2], data[3]) elif isinstance(data, int): t = data self.metrics['steps'].append(t * args.action_repeat + ( 0 if len(self.metrics['steps']) == 0 else self.metrics['steps'][-1])) self.metrics['episodes'].append(i + 1) else: print( "The Recvive Data Have Some Problems, Need To Fix") end_time = time.time() print("the process times {} s".format(end_time - start_time)) print( "End Multi Sample Processing -------------------------------")
def __init__(self): super().__init__() manager = Manager() self.send = manager.list() self.date = manager.list() self.quote = manager.list() self.number = manager.list() self.header = manager.list() self.sign = manager.list() self.device = torch.device('cpu') state_dict = torch.load( '/home/dung/Project/Python/ocr/craft_mlt_25k.pth') if list(state_dict.keys())[0].startswith("module"): start_idx = 1 else: start_idx = 0 new_state_dict = OrderedDict() for k, v in state_dict.items(): name = ".".join(k.split(".")[start_idx:]) new_state_dict[name] = v self.craft = CRAFT() self.craft.load_state_dict(new_state_dict) self.craft.to(self.device) self.craft.eval() self.craft.share_memory() self.config = Cfg.load_config_from_name('vgg_transformer') self.config[ 'weights'] = 'https://drive.google.com/uc?id=13327Y1tz1ohsm5YZMyXVMPIOjoOA0OaA' self.config['device'] = 'cpu' self.config['predictor']['beamsearch'] = False self.weights = '/home/dung/Documents/transformerocr.pth'