def file_in(self): self.statusbar.showMessage('正在导入数据,请稍等.........') fileDialog = QFileDialog(self, filter="文本文件 (*.txt)") fileDialog.setWindowModality(Qt.WindowModal) fileDialog.exec_() if len(fileDialog.selectedFiles()) != 0: txtPath = fileDialog.selectedFiles()[0] self.set_in_widget_visible(True) self.thread = Worker(txtPath, self.path, self.id_category_first_max, self.id_webAddress_max, self) self.thread.signal_change_value.connect( lambda value: self.progressBar.setValue(value)) self.thread.signal_update.connect(self.data_update) self.thread.signal_change_statusbar.connect( lambda message, time: self.statusbar.showMessage( message, time)) self.thread.signal_set_in_widget_visiable.connect( self.set_in_widget_visible) self.thread.signal_critical.connect(lambda: QMessageBox.critical( self, '错误', '文本内容格式错误,请选择正确文件', QMessageBox.Ok)) self.thread.start() else: self.statusbar.showMessage('')
def on_message(self, message): if len(message) > 4 and message[0:4] == "CODE": message = message.split('-') self.codeTransmitted = "" self.className = message[1] self.waitLinesCode = True elif message == "ENDCODE": self.waitLinesCode = False print(self.codeTransmitted) with open("./module/process.py", 'w+', encoding='utf-8') as fd: fd.write(self.codeTransmitted) self.codeTransmitted = None elif message == "DATA": self.waitData = True self.data = "" elif message == "ENDDATA": self.waitData = False self.data = eval(self.data) print(self.data) elif message == "RUN": clazz = getattr(importlib.import_module("module.process"), self.className) process = clazz() self.worker = Worker(self.data, process, self) self.worker.start() elif self.waitData: self.data += message elif self.waitLinesCode: self.codeTransmitted += message + "\n"
def __performTests(self): EARepository = PopulationRepository() PSORepository = PopulationRepository() acoProblem = ACOProblem(7, 1000, 40, 1.9, 0.9, 0.05, 0.5) self.__evolutionaryController = EvolutionaryController( EARepository, 7, 40, 0.5, 1000) self.__hillClimbingController = HillClimbingController(7, 1000) self.__psoController = PSOController(PSORepository, 7, 100, 50, 0.5, 1.1, 1.5, 10) self.__acoController = ACOController(acoProblem) self.__readInputEvolutionary() self.__readInputHillClimbing() self.__readInputPSO() self.__readInputACO() worker = Worker(self.__doTheTests) #worker.signals.result.connect(self.__printTestResults) worker.signals.finished.connect(self.__enableButtons) self.__disableButtons() self.threadpool.start(worker)
def __init__(self): self.tableQueue = queue.Queue() self.keyBindQueue = queue.Queue() self.keyPressQueue = queue.Queue() self.apps = None QtCore.QThread.currentThread().setObjectName("MAIN") self.thread = QtCore.QThread() self.thread.name = "auto_refresh" self.worker = Worker(self.tableQueue) self.worker.moveToThread(self.thread) self.worker.update_table.connect(self.updateTable) self.threadTwo = QtCore.QThread() self.threadTwo.name = "key_searcher" self.keyboardShortcuts = keyboardShortcuts(self.keyBindQueue, self.keyPressQueue) self.keyboardShortcuts.moveToThread(self.threadTwo) self.keyboardShortcuts.new_key_press.connect(self.ChangeVolume) self.session = [] self.appKeyMap = {} self.comboMap = {} self.keyboardShortcuts.start() self.worker.start()
def start(self): tags = self.lineEdit_tags.text() path = self.lineEdit_savepath.text() start_page = self.lineEdit_startpage.text() if start_page == '': start_page = 1 self.thread = Worker(tags, path, start_page, token, chan_cookies) self.thread.set_processbar.connect(self.set_processbar) self.thread.log_append.connect(self.log_append) self.thread.log_moveCursor.connect(self.log_moveCursor) self.thread.enable_start_button.connect(self.enable_start_button) self.thread.disable_start_button.connect(self.disable_start_button) self.thread.enable_stop_button.connect(self.enable_stop_button) self.thread.disable_stop_button.connect(self.disable_stop_button) self.thread.set_infolabel.connect(self.set_infolabel) self.thread.set_speedlabel.connect(self.set_speedlabel) self.thread.set_startbutton_text.connect(self.set_startbutton_text) self.thread.info_message.connect(self.info_message) self.thread.notifications.connect(self.notifications) self.thread.set_process.connect(self.set_process) self.thread.save_failed_list.connect(self.save_failed_list) self.thread.start()
# create and start threads threads = [] n_threads = 4 fnos = list(range(0, 3000, 10)) n_threads = 1 # n_threads is the number of worker threads to read video frame tasks = [[] for _ in range(0, n_threads)] # store frame number for each threads frame_per_thread = math.ceil(len(fnos) / n_threads) tid = 0 for idx, fno in enumerate(fnos): tasks[math.floor(idx / frame_per_thread)].append(fno) for _ in range(0, n_threads): w = Worker() threads.append(w) w.start() results = queue.Queue(maxsize=100) on_done = lambda x: results.put(x) # distribute the tasks from main to worker threads for idx, w in enumerate(threads): w.decode(video_path, tasks[idx], on_done) def analyze_gaze(frame): gaze.refresh(frame) frame = gaze.annotated_frame() text = ""
from functions import handle_new_page def initialize_logger(log_level: str): logging_level = logging.getLevelName(log_level) logging_format = '%(asctime)s %(module)s:%(lineno)d %(levelname)-8s %(message)s' logging.basicConfig(level=logging_level, format=logging_format) if __name__ == "__main__": cm = ConfigManager() crawler_config = cm.get_crawler_params() initialize_logger(crawler_config['log_level']) logging.warning('Running with params:') pprint(crawler_config) seed_urls = crawler_config['seed_urls'] coredb = CoreDb() for url in seed_urls: handle_new_page(coredb, url, crawler_config, Worker.locks) threads = list() for index in range(crawler_config['workers']): x = Worker(crawler_config) threads.append(x) x.start() for index, thread in enumerate(threads): thread.join()
#!/usr/bin/env python3 from Worker import Worker from time import sleep def function(inputs): sleep(3) return {"res": inputs['key2'] * 2} worker = Worker(5, 1) worker.start(function)
name='global_episodes', trainable=False) trainer = tf.train.RMSPropOptimizer(learning_rate=7e-4, epsilon=0.1, decay=0.99) master_network = Network(height, width, depth, s_size, a_size, 'global', None) # Generate global network num_workers = multiprocessing.cpu_count( ) # Set workers ot number of available CPU threads print 'Creating', num_workers, 'workers' workers = [] # Create worker classes for i in range(num_workers): workers.append( Worker(game_env, i, (height, width, depth, s_size), a_size, trainer, model_path, global_episodes)) saver = tf.train.Saver(max_to_keep=5) with tf.Session() as sess: coord = tf.train.Coordinator() if load_model: print('Loading Model...') ckpt = tf.train.get_checkpoint_state(model_path) saver.restore(sess, ckpt.model_checkpoint_path) else: sess.run(tf.global_variables_initializer()) # This is where the asynchronous magic happens. # Start the "work" process for each worker in a separate threat. worker_threads = []
#!/usr/bin/env python from Worker import Worker from time import sleep def function(inputs): sleep(3) return { "res": inputs['key2'] * 2 } worker = Worker(5,) worker.start(function)
def updateBerita(self, beginDate, endDate): """ Tambah data entri berita Input: beginDate -- integer Awal tanggal pencarian berita dengan format beginDate = beginyear * 10000 + beginMonth * 100 + beginDay endDate -- integer Akhir tanggal pencarian berita dengan format beginDate = endyear * 10000 + endMonth * 100 + endDay Output: Sebuah integer menunjukan berapa banyak data berita yang ditambah """ if beginDate > endDate: raise ValueError( "beginDate (%d) harus secara kronologis sebelum endDate (%d)" % (beginDate, endDate)) beginDate = datetime.datetime(beginDate // 10000, beginDate // 100 % 100, beginDate % 100) endDate = datetime.datetime(endDate // 10000, endDate // 100 % 100, endDate % 100) for I in FileParser.FileParser.parserList: parser = FileParser.FileParser(I) print(I) date = beginDate worker = Worker() while date <= endDate: newsList = parser.extractPublikasi(date.year * 10000 + date.month * 100 + date.day) print(date.year * 10000 + date.month * 100 + date.day) for J in newsList: #Cek apakah artikel sudah pernah di-crawl? self.c.execute("SELECT * FROM berita where Url = ?", (J['url'], )) d = self.c.fetchall() #Kalau belum, masukan if len(d) == 0: worker.addOrder(J['url']) result = worker.getData() print(len(result)) for J in newsList: if J['url'] in result: r = result[J['url']] nextTwoSession = self.cariSesi(J['tanggal'], J['jam']) # Sentimen positif = 1 # Sentimen netral = 0 # Sentimen negatif = -1 # Untuk awal, diasumsikan semua berita ketika harga naik adalah sentimen # positif dan diasumsikan semua berita ketika harga turun adalah # sentimen negatif sentiment = int( self.bandingIndeks(nextTwoSession) > 0) * 2 + -1 d = parser.extractData(r) if d != None: self.c.execute( "INSERT INTO berita VALUES (?,?,?,?,?,?,?,?)", (J['sumber'], J['judul'], J['url'], d, '', J['tanggal'], J['jam'], sentiment)) date += datetime.timedelta(1) worker.reset() self.conn.commit()
''' @author: vbegun ''' from Worker import Worker if __name__ == '__main__': pass bob = Worker('Bob Smith', 50000) print(bob.getPay())
if PREVIOUS_SESSION: for i, (name, net) in enumerate(zip(workernames, networks)): env = CONFIG.env saver_ = saver if i is not 0: saver_ = None env = getEnv(CONFIG.GAME, CONFIG.INIT_PORT + i, CONFIG.N_A, CONFIG.ACTION_REPEAT, CONFIG.RANDOM_SEED) else: print("first worker, we reuse existing env used for init") workers.append( Worker(name, net, SESS, COORD, CONFIG, unique_mutex, summary_writer, env=env, saver=saver_)) else: for i, (name, net) in enumerate(zip(workernames, networks)): env = CONFIG.env saver_ = saver if i is not 0: saver_ = None env = getEnv(CONFIG.GAME, CONFIG.INIT_PORT + i + 1, CONFIG.N_A, CONFIG.ACTION_REPEAT, CONFIG.RANDOM_SEED) workers.append( Worker(name, net,
#!/usr/local/bin/python3 from Worker import Worker if __name__ == '__main__': worker = Worker() worker.consume()
def add_worker(self): worker = Worker(len(self.workers), self) self.workers.append(worker) self.workers[-1].start() return worker
grad_queue = mp.Queue() rewards_queue = mp.Queue(maxsize=benchmark_freq) rewards_list = [] # Worker Params n_steps = 5 frame_dimensions = [84, 84] gamma = 0.99 env_name = "MsPacman-v0" print("Starting to create {} workers...".format(num_workers)) workers = [] for i in range(num_workers): color = colours[i % len(colours)] worker_id = "worker_{}".format(i + 1) worker = Worker(worker_id, env_name, network, n_steps, gamma, color, grad_queue, rewards_queue) workers.append(worker) threads = [] for worker in workers: t = mp.spawn(run, args=(worker, ), nprocs=1, join=False, daemon=False, start_method='spawn') threads.append(t) grads = None grad_count = 0 batch_size = 64
summary_writer = [] for id in range(n_threads): summary_writer.append(tf.summary.FileWriter(args.summary_dir+'/worker_'+str(id))) summary_parameters = Summary_Parameters() write_op = tf.summary.merge_all() master_network = A3C_Network(args, no_action, 'master_network') workers = [] env_list = [] for id in range(n_threads): env = gym.make(args.environment) if id == 0: env = gym.wrappers.Monitor(env, "monitors", force=True) workers.append(Worker(global_episodes, training_episodes, master_network, id, learning_rate, env, summary_writer[id], summary_parameters, write_op, args)) env_list.append(env) with tf.Session() as sess: saver = tf.train.Saver(max_to_keep=5) master_network.load_model(sess, saver) coord = tf.train.Coordinator() thread_list = [] for id in range(n_threads): t = threading.Thread(target=workers[id].process, args=(sess, coord, saver)) t.start() sleep(0.5) thread_list.append(t) coord.join(thread_list)
if __name__ == "__main__": startTime = time.time() #conf = SparkConf().setMaster("local").setAppName("Test") conf = SparkConf().setMaster("spark://localhost:7077").setAppName("Test") # sc = SparkContext(conf=conf, pyFiles=["/home/crist/PycharmProjects/DistributedTest/dependencies.zip"]) sc = SparkContext(conf=conf) scenarioSize = 20 workerList = [] scenarioList = [] for i in range(scenarioSize): workerList.append(Worker(i)) scenarioList.append("Scenario " + str(i)) parallelWorkerList = sc.parallelize(zip(workerList, scenarioList)).persist() for x in range(20): var1 = randint(100, 10000) var2 = randint(100, 10000) print("Initializing iteration " + str(x)) print("Updating variables: ") print("\tvar1: " + str(var1)) print("\tvar2: " + str(var2)) broadcast1 = sc.broadcast(var1)
momentum=0.9) lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=8000, gamma=0.1) # resume from last time if necessary: epoch_offset = 0 step_offset = 0 if ckpt_dir: ckpt = torch.load(ckpt_dir, map_location=device) epoch_offset = ckpt['epoch_offset'] step_offset = ckpt['step_offset'] model.load_state_dict(ckpt['state_dict']) if optim_load: optimizer.load_state_dict(ckpt['optim']) print('resume from epoch: {}, step: {}'.format(epoch_offset - 1, step_offset - 1)) # just before work: setup logger log_folder = log_folder if log_folder else str( datetime.now().strftime('%Y-%m-%d_%H:%M:%S')) writer = SummaryWriter(log_root + '/{}'.format(log_folder) ) # TensorboardX: https://zhuanlan.zhihu.com/p/35675109 # work, the main loop! worker = Worker(model, optimizer, lr_scheduler, criterion, epochs, epoch_offset, step_offset, train_loader, val_loader, test_loader, device, writer, log_root, log_folder) print(worker.work())
with tf.device("/cpu:0"): global_episodes = tf.Variable(0, dtype=tf.int32, name='global_episodes', trainable=False) trainer = tf.train.AdamOptimizer(learning_rate=1e-4) master_network = AC_Network(s_size, a_size, 'global', None) # Generate global network num_workers = multiprocessing.cpu_count( ) # Set workers to number of available CPU threads workers = [] # Create worker classes for i in range(int(num_workers / 2)): workers.append( Worker(Environment(), i, s_size, a_size, trainer, model_path, global_episodes)) saver = tf.train.Saver(max_to_keep=5) with tf.Session() as sess: coord = tf.train.Coordinator() if load_model == True: print('Loading Model...') ckpt = tf.train.get_checkpoint_state(model_path) saver.restore(sess, ckpt.model_checkpoint_path) else: sess.run(tf.global_variables_initializer()) # Start the "work" process for each worker in a separate threat. worker_threads = [] for worker in workers: worker_work = lambda: worker.work(max_episode_length, gamma, sess,
def __init__(self, conf_file): self.worker = Worker(conf_file)
def multi_solve_environment(self): workers_top20 = [] for arch_epoch in range(self.arch_epochs): results_queue = Queue() processes = [] for episode in range(self.episodes): actions_p, actions_log_p, actions_index = self.controller.sample( ) actions_p = actions_p.cpu().numpy().tolist() actions_log_p = actions_log_p.cpu().numpy().tolist() actions_index = actions_index.cpu().numpy().tolist() if episode < self.episodes // 3: worker = Worker(actions_p, actions_log_p, actions_index, self.args, 'cuda:0') elif self.episodes // 3 <= episode < 2 * self.episodes // 3: worker = Worker(actions_p, actions_log_p, actions_index, self.args, 'cuda:1') else: worker = Worker(actions_p, actions_log_p, actions_index, self.args, 'cuda:3') process = Process(target=consume, args=(worker, results_queue)) process.start() processes.append(process) for process in processes: process.join() workers = [] for episode in range(self.episodes): worker = results_queue.get() worker.actions_p = torch.Tensor(worker.actions_p).to( self.device) worker.actions_index = torch.LongTensor( worker.actions_index).to(self.device) workers.append(worker) for episode, worker in enumerate(workers): if self.baseline == None: self.baseline = worker.acc else: self.baseline = self.baseline * self.baseline_weight + worker.acc * ( 1 - self.baseline_weight) # sort worker retain top20 workers_total = workers_top20 + workers workers_total.sort(key=lambda worker: worker.acc, reverse=True) workers_top20 = workers_total[:20] top1_acc = workers_top20[0].acc top5_avg_acc = np.mean( [worker.acc for worker in workers_top20[:5]]) top20_avg_acc = np.mean([worker.acc for worker in workers_top20]) logging.info( 'arch_epoch {:0>3d} top1_acc {:.4f} top5_avg_acc {:.4f} top20_avg_acc {:.4f} baseline {:.4f} ' .format(arch_epoch, top1_acc, top5_avg_acc, top20_avg_acc, self.baseline)) for i in range(5): print(workers_top20[i].genotype) for ppo_epoch in range(self.ppo_epochs): loss = 0 for worker in workers: actions_p, actions_log_p = self.controller.get_p( worker.actions_index) loss += self.cal_loss(actions_p, actions_log_p, worker, self.baseline) loss /= len(workers) logging.info('ppo_epoch {:0>3d} loss {:.4f} '.format( ppo_epoch, loss)) self.adam.zero_grad() loss.backward() self.adam.step()
default=10, help="Update the Tensorboard every X steps.") args = parser.parse_args() tf.reset_default_graph() model_path = os.path.join('./model/', args.game_name) if not os.path.exists(model_path): os.makedirs(model_path) # create openai game env = gym.make(args.game_name) n_actions = env.action_space.n global_frame_counter = 0 with tf.Session() as sess: with tf.device("/cpu:0"): # create global network global_network = A3C_Network( args, n_actions, trainer=None, scope=GLOBAL_NETWORK_NAME) # Generate global network worker = Worker(args, 0, model_path, None, global_network, None, None, None) sess.run(tf.global_variables_initializer()) worker.evaluate(sess)
def start(self): workers = [] network_params = (self.NUM_STATE, self._config['Max steps taken per batch'], self.NUM_ACTIONS, self.ACTION_SPACE) # Init Global and Local networks. Generate Weights for them as well. if self._config['CNN type'] == '': self._global_model = AC_Model_Large(self.NUM_STATE, self.NUM_ACTIONS, self._config, is_training=True) self._global_model( tf.convert_to_tensor(np.random.random((1, *self.NUM_STATE)), dtype='float64')) self._step_model = AC_Model_Large(self.NUM_STATE, self.NUM_ACTIONS, self._config, is_training=True) self._step_model( tf.convert_to_tensor(np.random.random((1, *self.NUM_STATE)), dtype='float64')) else: self._global_model = CNN_class_import( self._config['CNN type'], (self.NUM_STATE, self.NUM_ACTIONS, self._config, True)) self._global_model( tf.convert_to_tensor(np.random.random((1, *self.NUM_STATE)), dtype='float64')) self._step_model = CNN_class_import( self._config['CNN type'], (self.NUM_STATE, self.NUM_ACTIONS, self._config, True)) self._step_model( tf.convert_to_tensor(np.random.random((1, *self.NUM_STATE)), dtype='float64')) # Load model if exists if not os.path.exists(self._model_save_path): os.makedirs(self._model_save_path) else: try: if (os.path.exists(self._model_save_path + "\checkpoint")): self._global_model.load_model_weights() self._step_model.load_model_weights() for env in self._envs: workers.append( Worker(self._step_model, env, batch_size=self. _config['Max steps taken per batch'], render=False)) print("Model restored.") else: for env in self._envs: workers.append( Worker(self._step_model, env, batch_size=self. _config['Max steps taken per batch'], render=False)) print("Creating new model.") except: print("ERROR: There was an issue loading the model!") raise coordinator = Coordinator(self._global_model, self._step_model, workers, self._plot, self._model_save_path, self._config) # Train and save try: if coordinator.run(): try: self._global_model.save_model_weights() print("Model saved.") return True except: print("ERROR: There was an issue saving the model!") raise except: print("ERROR: There was an issues during training!") raise
def __init__(self, num_threads): self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks)
def __init__(self): self._crewId = None self._worker1 = Worker() self._worker2 = Worker() self._worker3 = Worker()
weight_path = "model/a3c-{}-weight".format(env_name.lower()) load_weights = True max_episode_each_worker = 50000 per_global_episode_play = 25 num_episode_play = 5 gamma = 0.99 # Future reward discount rate. alpha_actor = 0.00001 # Learning rate for actors. alpha_critic = 0.0001 # Learning rate for critics. sess = tf.Session() with sess.as_default(), sess.graph.as_default(): # Create main agent. MainAgent = Worker("Main", env_name, gamma, alpha_actor, alpha_critic, sess) # Create workers. Workers = [ Worker("Worker-{}".format(w_id), env_name, gamma, alpha_actor, alpha_critic, sess) for w_id in range(1, num_workers+1) ] global_episode = tf.Variable(0.0, dtype=tf.float32, name="global_episode", trainable=False) global_episode_increment_name = global_episode.assign_add(1.0, name="global_episode_increment").name coord = tf.train.Coordinator() sess.run(tf.global_variables_initializer()) # Only save main weights to disk. # Workers' initial weights are already copied from main network.
def setUp(self): self.worker = Worker(conf_file) self.config = self.worker.config self.tasks = self.config.sections() self.last_tasks = {}
def create_new_worker(): create_new_worker.num_of_created += 1 new_worker = Worker(name="worker_" + str(create_new_worker.num_of_created)) return Process(target=new_worker.run), "worker_" + str(create_new_worker.num_of_created)
if ENV_NAME == "Breakout-v0": image_crop = [31, 195] elif ENV_NAME == "MsPacman-v0": image_crop = [0, -50] else: image_crop = [0, -0] print("Starting to create {} workers...".format(num_workers)) workers = [] for i in range(num_workers): color = colours[i % len(colours)] worker_id = "worker_{}".format(i + 1) image_processor = GameImageProcessor(image_crop, frame_dimensions) worker = Worker(worker_id, ENV_NAME, network, n_steps, gamma, color, grad_queue, rewards_queue, image_processor) workers.append(worker) threads = [] for worker in workers: t = mp.spawn(run, args=(worker, ), nprocs=1, join=False, daemon=False, start_method='spawn') threads.append(t) grads = None grad_count = 0 batch_size = 64