def run_parkinglot_expt(net, n): "Run experiment" seconds = args.time # Start the bandwidth and cwnd monitors in the background monitor = Process(target=monitor_devs_ng, args=('%s/bwm.txt' % args.dir, 1.0)) monitor.start() start_tcpprobe() # Get receiver and clients recvr = net.getNodeByName('receiver') #sender1 = net.getNodeByName('h1') # Start the receiver port = 5001 recvr.cmd('iperf -s -p', port, '> %s/iperf_server.txt' % args.dir, '&') #waitListening(sender1, recvr, port) # TODO: start the sender iperf processes and wait for the flows to finish # Hint: Use getNodeByName() to get a handle on each sender. # Hint: Use sendCmd() and waitOutput() to start iperf and wait for them to finish # Hint: waitOutput waits for the command to finish allowing you to wait on a particular process on the host # iperf command to start flow: 'iperf -c %s -p %s -t %d -i 1 -yc > %s/iperf_%s.txt' % (recvr.IP(), 5001, seconds, args.dir, node_name) # Hint (not important): You may use progress(t) to track your experiment progress ### begin my code # get list of all hosts h = [] # Python list of hosts for i in range(n): #print "DEBUG", 'h%s' % (i+1) h.append(net.getNodeByName('h%s' % (i + 1))) # wait for ports on all iperf clients for i in range(n): waitListening(h[i], recvr, port) # send iperf cmd to all hosts for i in range(n): node_name = 'h%s' % (i + 1) h[i].sendCmd('iperf -c %s -p %s -t %d -i 1 -yc > %s/iperf_%s.txt' % (recvr.IP(), port, seconds, args.dir, node_name)) # wait for commands to finish iperf_results = {} progress(seconds) # show progress while waiting for i in range(n): iperf_results[h[i].name] = h[i].waitOutput() ### end my code recvr.cmd('kill %iperf') # Shut down monitors monitor.terminate() stop_tcpprobe()
def process_message(m, local_message_i, max_run_time, run_dir, aws_region, server_name, log_stream_prefix): event = json.loads(m.body) # run this in a thread: pywren.wrenhandler.generic_handler(event) p = Process(target=job_handler, args=(event, local_message_i, run_dir, aws_region, server_name, log_stream_prefix)) # is thread done p.start() start_time = time.time() response = m.change_visibility( VisibilityTimeout=SQS_VISIBILITY_INCREMENT_SEC) # add 10s to visibility run_time = time.time() - start_time last_visibility_update_time = time.time() while run_time < max_run_time: if (time.time() - last_visibility_update_time) > ( SQS_VISIBILITY_INCREMENT_SEC * 0.9): response = m.change_visibility( VisibilityTimeout=SQS_VISIBILITY_INCREMENT_SEC) last_visibility_update_time = time.time() logger.debug("incrementing visibility timeout by {} sec".format( SQS_VISIBILITY_INCREMENT_SEC)) if p.exitcode is not None: logger.debug("attempting to join process") # FIXME will this join ever hang? p.join() break else: print "sleeping" time.sleep(PROCESS_SLEEP_DUR_SEC) run_time = time.time() - start_time if p.exitcode is None: p.terminate() # PRINT LOTS OF ERRORS HERE m.delete()
## Start weewx engine with options/args and pass in shared data dict p_weewx = Process(target=weewx.engine.main, args=(options, args, sdata)) p_weewx.start() ## ------------------------------------------------------------- # Forever loop for data transform, check process health last_ts = sdata["t"] # Timestamp last_clear_time = last_ts while True: time.sleep(0.1) # Weather data post interval >= 2.5s, so take it easy. # Check for new weather data from weewx (will be a string, not dict) if sdata["t"] != last_ts and type(sdata["r"]) == str: # Transform weather data to dict sdata["r"] = format_weather_data(copy.deepcopy(sdata["r"])) sdata["r"]["status"] = "Station running..." last_ts = sdata["t"] # Check processes and kill this script if one dies if not p_weewx.is_alive(): logger.error("start.py: weewx process died: exit code = %s\n" % (p_weewx.exitcode)) p_flask.terminate() sys.exit(1) if not p_flask.is_alive(): logger.error("start.py: flask process died: exit code = %s\n" % (p_flask.exitcode)) p_weewx.terminate() sys.exit(1) logger.error("start.py: Script exited from run loop unexpectedly") sys.exit(1)
print('Process to write: %s' % os.getpid()) for value in ['A', 'B', 'C','D']: print('Put %s to queue...' % value) q.put(value) l.append(value) print l time.sleep(random.random()) # 读数据进程执行的代码: def read(q,l): print('Process to read: %s' % os.getpid()) while True: value = q.get(True) print('Get %s from queue.' % value) print l if __name__=='__main__': # 父进程创建Queue,并传给各个子进程: q = Queue() l=[] pw = Process(target=write, args=(q,l)) pr = Process(target=read, args=(q,l)) # 启动子进程pw,写入: pw.start() # 启动子进程pr,读取: pr.start() # 等待pw结束: pw.join() # pr进程里是死循环,无法等待其结束,只能强行终止: pr.terminate()
class _Task(object): def __init__(self, name, weight, target, args=None, kwargs=None, daemon=None, force_quit=False): self.name = name self.weight = 0 - weight self._target = target self._args = args self._kwargs = kwargs self.force_quit = force_quit self._daemon = True self._lock = threading.Lock() self._task = None self._stop = False self._start_new() def _start_new(self): if self._task is not None and self._task.is_alive(): logging.info(" %s is alive, no need to start new" % self.name) return self._task = Process(target=self._target, name=self.name, args=self._args, kwargs=self._kwargs, daemon=self._daemon) if isinstance(self._args[0], Args): logging.info("delete %s", self._args[0].export_path) if gfile.Exists(self._args[0].export_path): logging.info(" deleting") gfile.DeleteRecursively(self._args[0].export_path) self._task.start() logging.info("Task starts %s" % self.name) time.sleep(10) def __gt__(self, o): return self.weight > o.weight def kill(self, force=False): with self._lock: logging.info("Kill task %s", self.name) if self._task is None or not self._task.is_alive(): return if force or self.force_quit: self._task.terminate() elif self._task.is_alive(): raise ValueError("can not kill by force") def start(self): logging.info("begin to start") with self._lock: if self._stop: return if self._task.is_alive(): logging.info(" %s is alive, no need to start" % self.name) return self._start_new() time.sleep(2) def is_alive(self): with self._lock: if self._task is None: return True return self._task.is_alive()
class Job: def __init__(self, args): self.prefix = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) self.file = args.file self.in_dir = args.in_dir if args.in_dir else args.turkey_home self.out_dir = args.out_dir if args.out_dir is not None else \ os.path.join(args.turkey_home, 'out', self.prefix + '_' + self.file.split('/')[-1].split('.')[0] + '.out') os.system('mkdir -p %s' % self.out_dir) # Copy over job file to out directory for easy parsing later os.system('cp %s %s' % (self.file, self.out_dir)) with open(self.file, 'r') as f: self.tasks = [ Task(task, out_dir=self.out_dir, in_dir=self.in_dir, turkey_home=args.turkey_home) for task in json.load(f) ] self.pool_size = args.pool_size self.num_cpus = args.num_cpus # TODO: might want a better name (i.e., moldable Linux) self.intelligent = args.intelligent def run(self, stdout=False): # Set up up system stat collector self.stat_process = Process(target=write_stats, args=(os.path.join(self.out_dir, 'stats.csv'), )) self.stat_process.start() pool_size = min(len(self.tasks), self.pool_size, mp.cpu_count()) pool = mp.Pool(pool_size) args = {} # Initialize the number of tasks remaining config.num_tasks_remaining.set(len(self.tasks)) for task in self.tasks: # TODO: We should discuss the details of intelligent. For example, in # the case where tasks are pinned to fewer than cpu_count number of # cores if self.intelligent: args['nthreads'] = int(self.num_cpus / config.num_tasks_in_system) # Wait before we deliver the next task # TODO: Not great that this happens on the main thread task.delay() apply_args = { 'args': args, 'stdout': stdout, 'wait': True, 'count': True } pool.apply_async(task.run, (), apply_args) # Normally we'd use os.wait(), but between wanting to wait for the # async applies to finish and os.wait() also depending on the stat # process, we have a deadlock while config.num_tasks_remaining.value > 0: time.sleep(5) self.stat_process.terminate() os.system('stty sane')