def _return_debug_cluster_info(self, request, context=None) -> str: """Handle ClusterInfo requests that only return a json blob.""" data = None if request.type == ray_client_pb2.ClusterInfoType.NODES: data = ray.nodes() elif request.type == ray_client_pb2.ClusterInfoType.IS_INITIALIZED: data = ray.is_initialized() elif request.type == ray_client_pb2.ClusterInfoType.TIMELINE: data = ray.timeline() else: raise TypeError("Unsupported cluster info type") return json.dumps(data)
def train_remote(X, y, num_processes, batch_size): ray.init(num_cpus=6) X_parts = np.array_split(X, num_processes) y_parts = np.array_split(y, num_processes) X_ids = [ray.put(X_part) for X_part in X_parts] y_ids = [ray.put(y_part) for y_part in y_parts] ps = AdamParameterServer.remote(dim=X.shape[1], lr=.1) start_time = time.time() workers = [ gradient_worker.remote(ps, X_ids[i], y_ids[i], batch_size) for i in range(num_processes) ] worker_res = ray.get(workers) end_time = time.time() print("Elapsed Time: {}".format(end_time - start_time)) res = ray.get(ps.get_params.remote()) ray.timeline("timeline.json") ray.shutdown() return res
def test_profiling_api(ray_start_2_cpus): @ray.remote def f(): with ray.profile("custom_event", extra_data={"name": "custom name"}): pass ray.put(1) object_ref = f.remote() ray.wait([object_ref]) ray.get(object_ref) # Wait until all of the profiling information appears in the profile # table. timeout_seconds = 20 start_time = time.time() while True: profile_data = ray.timeline() event_types = {event["cat"] for event in profile_data} expected_types = [ "task", "task:deserialize_arguments", "task:execute", "task:store_outputs", "wait_for_function", "ray.get", "ray.put", "ray.wait", "submit_task", "fetch_and_run_function", # TODO (Alex) :https://github.com/ray-project/ray/pull/9346 # "register_remote_function", "custom_event", # This is the custom one from ray.profile. ] if all(expected_type in event_types for expected_type in expected_types): break if time.time() - start_time > timeout_seconds: raise RayTestTimeoutException( "Timed out while waiting for information in " "profile table. Missing events: {}.".format( set(expected_types) - set(event_types))) # The profiling information only flushes once every second. time.sleep(1.1)
param_server = ParameterServer.remote() # buffer if use_per: buffer = PrioritizedReplayBuffer.remote(size=buffer_max_size, alpha=priority_alpha) else: buffer = Buffer.remote(max_size=buffer_max_size) # learner: learner = Learner.remote(config, buffer, param_server, experiment_dir) param_list = ray.get(learner.return_param_list.remote()) param_server.define_param_list.remote(param_list) learner.update_param_server.remote() # actors actors = [Actor.remote(i, config, experiment_dir, buffer, param_server) for i in range(n_agents)] # processes procs = [] procs.append(learner) for actor in actors: procs.append(actor) print("run") ray.wait([proc.run.remote() for proc in procs]) ray.timeline() print("End")
def job_async(num_tasks, n): task_result_ids = [task_async.remote() for task_id in range(num_tasks)] task_results = ray.get(task_result_ids) ray.timeline(filename="timeline.json") return sum(task_results)
from tensorflow.keras.applications import ResNet50 from dkeras import dKeras import numpy as np import ray ray.init() data = np.random.uniform(-1, 1, (100, 224, 224, 3)) model = dKeras(ResNet50, init_ray=False, wait_for_workers=True, n_workers=4) preds = model.predict(data) ray.timeline('test') """ int8: 155.15053375276426 6.444483995437622 155.17146147122887 """
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',filename='app.log',level=logging.DEBUG) print(os.getpid()) print(os.getppid()) if not ray.is_initialized(): ray.init(include_webui=True) files = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M'] db.setup() print(os.getpid()) print(os.getppid()) with ray.profile('Event'): for i in range(10): time.sleep(randint(0, 4)) try: ray.get(worker.remote(i)) except Exception as e: raise e print(e.message) finally: print('finally') #results = [worker.remote(file) for file in (files)] #ray.get(results) if __name__ == '__main__': main() ray.timeline(filename='timeline.dump') print(ray.errors())
do_tune=False, do_sacred=False, do_tqdm=True, do_exit=False) else: kwargs = {'num_cpus': args.n_cpus} if args.n_cpus == 0: kwargs = {'num_cpus': 1, 'local_mode': True} ray.init(**kwargs, num_gpus=args.n_gpus, include_dashboard=True) if args.resume: gin.bind_parameter('tune_run.resume', True) if args.load_checkpoint: gin.bind_parameter('Config.load_checkpoint', args.load_checkpoint) gin.bind_parameter('Config.load_new_config', args.load_new_config) if args.epochs_override is not None: def set_train_steps(): gin.bind_parameter('Config.train_steps', args.epochs_override) config.append(set_train_steps) # TODO: remove heavy metrics s.a. weights in experiment_state.json # gin.bind_parameter('tune_run.loggers', ) learner_gin_sacred(config, nofail=args.nofail) if args.ray_debug_timeline: ray.timeline(filename=args.ray_debug_timeline)
def export_timeline(): timestr = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") filename = f"/tmp/ray-timeline-{timestr}.json" ray.timeline(filename=filename) logging.info(f"Exported Ray timeline to {filename}")
write_rois(rass_data.output('v_%s.txt' % treatment_name, subfolder="%s" % treatment_name), totalDoses, roi_marks, kmax, jmax, imax, plan_grid_ct, v2Drow) if options["debug_beam_doses"]: vmc.saveToVTI(rass_data.output("mcp_doses"), mcDosesVMC, [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) if options["out_mc_doses"]: vmc.saveToVTI(rass_data.output("mc_doses"), np.reshape(mcDoses, np.prod([imax, jmax, kmax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) if options["out_mc_doses_fluence"]: vmc.saveToVTI(rass_data.output("mc_doses_fluence"), np.reshape(mcDosesFluence, np.prod([imax, jmax, kmax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) npTotalDoses = np.array(totalDoses, dtype=np.float32) if options["out_difference"]: vmc.saveToVTI(rass_data.output("npTotalDoses"), np.reshape(npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) vmc.saveToVTI(rass_data.output("difference"), np.reshape(mcDosesFluence-npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) if options["out_total_doses"]: log.info("Max total npTotalDoses: %f" % np.max(npTotalDoses)) vmc.saveToVTI(rass_data.output("totalDoses"), np.reshape(npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase]) ray.timeline("/tmp/ray-timeline.json") if options["histograms"]: def generate_gnuplot_histograms(name, subfolder, doses_to_plot): stats_fname = rass_data.output(f"statistics_{name}.txt"); png_fname = '%s_histogram_%s.png' % (treatment_name, name) log.info(f"Generating histograms for {name}... (statistics will be saved to: {stats_fname})") roi_statistics = {} fout = open(stats_fname,'w') f = open(rass_data.output('histograms.gpt', subfolder=subfolder), 'w') f.write('set grid\nset style data lp\nset xlabel \'Dose [cGy]\'\n' 'set ylabel \'%% of volume\'\nset yrange [0:110]\nset term png size 1024,768\nset output "%s"\nplot ' % png_fname) for r in range(len(myROIs)):
def main(): ray.init(address='10.155.1.10:6379', redis_password='******') ray.timeline(filename="timeline.json") start_time = time.time() var_names = [ 'beta', 'theta', 'p', 'sigma', 'rho', 'epsilonA', 'epsilonI', 'lambda0', 'gammaA', 'gammaI', 'gammaD', 'cD', 'dD', 'dI', 'delta', 'S0', 'R0', 'Q0', 'E0', 'A0', 'I0', 'D0', 'day' ] paramp = pp.ParamProcessor.remote('memory-edo', var_names) workload = dt.build_database(False) workerlist = list() cpu = 0 for node in ray.nodes(): cpu += int(node['Resources']['CPU']) #filter_list = ['Goias', 'Sao_Paulo', 'Egypt', 'Ethiopia', 'France'] #filter_list = dt.brasilian_regions() filter_list = workload.keys() apply_filter(workload, filter_list) print(f'Workload is {len(workload)}') work_chunk = submit_work(workload, filter_list, workerlist, paramp, program, cpu + 4) print( f'Submitting {work_chunk} tasks ({len(workload)}/{len(workerlist)} to go).' ) check_size = 4 with open('log/processed.log', 'w') as f: loops = 0 while len(workerlist) > 0: num_returns = check_size if len(workerlist) >= check_size else len( workerlist) ready, not_ready = ray.wait(workerlist, num_returns=num_returns) done = ray.get(ready) printx(3, 0, f'Iteration : {loops}') printx(4, 0, f'Ready length, regions : {len(ready)}') printx(5, 0, f'Not Ready length : {len(not_ready)}') printx(5, 0, f'Workload : {len(workload)}') printx(4, 40, 'Ready List') for pos, i in enumerate(done): printx(5 + pos, 40, f'{i}') elapsed_time = time.time() - start_time stetime = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) printx(7, 0, f'Elapsed time : {stetime}') for nct in done: f.write(f'{nct}\n') workerlist = not_ready if len(workerlist) < cpu: submit_work(workload, filter_list, workerlist, paramp, program, cpu - len(workerlist)) f.flush() time.sleep(15) loops += 1 elapsed_time = time.time() - start_time stetime = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) print(f'Loops to process = {loops} in {stetime}.') paramp.dump.remote('log/param.csv') return
def apex(remote_learner, remote_actors, main_buffer, remote_counter, remote_evaluate, log_wandb=False, max_eps=1000, replay_start_size=1000, batch_size=128, sync_nn_mod=100, number_of_batchs=16, beta=0.4, rollout_size=70, save_nn_mod=100, save_dir='train'): # Start tasks online_weights, target_weights = remote_learner.get_weights.remote() start_learner = False rollouts = {} for a in remote_actors: rollouts[a.rollout.remote(rollout_size, online_weights, target_weights)] = a rollouts[remote_evaluate.test.remote(save_dir, online_weights, target_weights)] = remote_evaluate episodes_done = ray.get(remote_counter.get_value.remote()) optimization_step = 0 priority_dict, ds = None, None # Main train process while episodes_done < max_eps: ready_ids, _ = ray.wait(list(rollouts)) first_id = ready_ids[0] first = rollouts.pop(first_id) if first == remote_learner: optimization_step += 1 start_time = timeit.default_timer() if optimization_step % sync_nn_mod == 0: save = (optimization_step % save_nn_mod == 0) * save_dir online_weights, target_weights = first.get_weights.remote(save) rollouts[first.update_from_ds.remote(ds, start_time, batch_size)] = first indexes, priorities = ray.get(first_id) indexes = indexes.copy() priorities = priorities.copy() main_buffer.update_priorities(indexes=indexes, priorities=priorities) ds = main_buffer.sample(number_of_batchs * batch_size, beta) elif first == remote_evaluate: score, eps_number = ray.get(first_id) if log_wandb: wandb.log({'Score': score, 'episode': eps_number}) rollouts[remote_evaluate.test.remote( save_dir, online_weights, target_weights)] = remote_evaluate else: rollouts[first.rollout.remote(rollout_size, online_weights, target_weights)] = first data, priorities = ray.get(first_id) priorities = priorities.copy() main_buffer.add(priorities=priorities, **data) if main_buffer.get_stored_size( ) > replay_start_size and not start_learner: start_time = timeit.default_timer() ds = main_buffer.sample(number_of_batchs * batch_size, beta) rollouts[remote_learner.update_from_ds.remote( ds, start_time, batch_size)] = remote_learner ds = main_buffer.sample(number_of_batchs * batch_size, beta) start_learner = True episodes_done = ray.get(remote_counter.get_value.remote()) ray.timeline()
simulator = Simulator(MODEL_GLCWB) results = {} for f_simulate in [po_bolus, iv_bolus, iv_infusion, clamp, mix, stepped_clamp]: f_key = f_simulate.__name__ print(f_key) result = f_simulate(simulator, r) print(result) print("results:", len(result), sys.getsizeof(result)) results[f_key] = result for key, result in results.items(): somatostatin_plot(result, title=key) plt.show() import ray ray.timeline(filename="timeline.json") ''' def exlude_pkdb_parameter_filter(pid): """ Returns True if excluded, False otherwise :param pid: :return: """ # TODO: implement # dose parameters if (pid.startswith("IVDOSE_")) or (pid.startswith("PODOSE_")): return True # physical parameters if (pid.startswith("Mr_")) or pid in ["R_PDB"]:
def main(): #pype.init_ray() ray.init() ray.timeline(filename="timeline.json") server = pype.Server.remote() server.add.remote('frame', use_locking=False) #server.add.remote('person', use_locking=False) # # server.add.remote('face', use_locking=False) server.add.remote('pose', use_locking=False) server.add.remote('object', use_locking=False) server.add.remote('action', use_locking=False) # server.add.remote('action', use_locking=False) # server.add.remote('action1', use_locking=False) # server.add.remote('counter', use_locking=False) # server.add.remote('results') person_models = [ NullDetector.remote(server, input_queue='frame', output_queues=['pose'], batch_size=-1) for _ in range(1) ] person_models = [ NullDetector.remote(server, input_queue='pose', output_queues=['object'], batch_size=-1) for _ in range(1) ] person_models = [ NullDetector.remote(server, input_queue='object', output_queues=['action'], batch_size=-1) for _ in range(1) ] # person_models = [NullDetector.remote(server, input_queue='action', # output_queues=['action1'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action1', # output_queues=['action2'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action2', # output_queues=['action3'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action3', # output_queues=['action4'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action4', # output_queues=['action5'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action5', # output_queues=['action6'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action6', # output_queues=['action7'], # batch_size=-1) for _ in range(1)] time.sleep(3) ray.timeline(filename="timeline.json") for _ in range(1): NullVideoServer.remote(server, output_queues='frame') # person_models = [NullDetector.remote(server, input_queue='action1', # output_queues=['action2'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action2', # output_queues=['action3'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action3', # output_queues=['action4'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action4', # output_queues=['action5'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action5', # output_queues=['action6'], # batch_size=-1) for _ in range(1)] # person_models = [NullDetector.remote(server, input_queue='action6', # output_queues=['action7'], # batch_size=-1) for _ in range(1)] # pose_models = [NullDetector.remote(server, input_queue='pose', # output_queues=['object']) for _ in range(1)] # face_models = [NullDetector.remote(server, input_queue='face', # output_queues=['results']) for _ in range(1)] # counter_models = [NullDetector.remote(server, input_queue='person', # output_queues=['results']) for _ in range(1)] # object_models = [NullDetector.remote(server, input_queue='object', # output_queues=['action']) for _ in range(1)] # action_models = [NullDetector.remote(server, input_queue='action', # output_queues=['results']) for _ in range(1)] ray.get(server.all_initalized.remote()) time_elapsed = 0 count = 0 start_time = time.time() ray.timeline(filename="timeline.json") while True: pype.pull_wait(server, 'action') data = ray.get(server.pull.remote('action')) #time_elapsed += time.time()-data[0] count += len(data) # print(count) if time.time() - start_time > 10: ray.timeline(filename="timeline.json") break print("Throughput: {}".format(count / (time.time() - start_time))) ray.shutdown()
def get(self): traceJson = ray.timeline() return traceJson
def chrome_tracing_dump(self, filename=None): logger.warning( "ray.global_state.chrome_tracing_dump() is deprecated and will be " "removed in a subsequent release. Use ray.timeline() instead.") return ray.timeline(filename=filename)
nargs=1, default=None, help= "directory to store the encrypted ballots on disk, enables equivalence checking (default: memory only)", ) parser.add_argument( "cvr_file", type=str, nargs="+", help="filename(s) for the Dominion-style ballot CVR file", ) args = parser.parse_args() files = args.cvr_file file_dir = args.dir[0] if args.dir else None use_progressbar = args.progress if args.local: print("Using Ray locally") ray_init_localhost() else: print("Using Ray on a cluster") ray_init_cluster() for arg in files: run_bench(arg, file_dir, use_progressbar) print("Writing Ray timelines to disk.") ray.timeline("ray-timeline.json") ray.object_transfer_timeline("ray-object-transfer-timeline.json")