def _return_debug_cluster_info(self, request, context=None) -> str:
"""Handle ClusterInfo requests that only return a json blob."""
data = None
if request.type == ray_client_pb2.ClusterInfoType.NODES:
data = ray.nodes()
elif request.type == ray_client_pb2.ClusterInfoType.IS_INITIALIZED:
data = ray.is_initialized()
elif request.type == ray_client_pb2.ClusterInfoType.TIMELINE:
data = ray.timeline()
else:
raise TypeError("Unsupported cluster info type")
return json.dumps(data)
def train_remote(X, y, num_processes, batch_size):
ray.init(num_cpus=6)
X_parts = np.array_split(X, num_processes)
y_parts = np.array_split(y, num_processes)
X_ids = [ray.put(X_part) for X_part in X_parts]
y_ids = [ray.put(y_part) for y_part in y_parts]
ps = AdamParameterServer.remote(dim=X.shape[1], lr=.1)
start_time = time.time()
workers = [
gradient_worker.remote(ps, X_ids[i], y_ids[i], batch_size)
for i in range(num_processes)
]
worker_res = ray.get(workers)
end_time = time.time()
print("Elapsed Time: {}".format(end_time - start_time))
res = ray.get(ps.get_params.remote())
ray.timeline("timeline.json")
ray.shutdown()
return res
def test_profiling_api(ray_start_2_cpus):
@ray.remote
def f():
with ray.profile("custom_event", extra_data={"name": "custom name"}):
pass
ray.put(1)
object_ref = f.remote()
ray.wait([object_ref])
ray.get(object_ref)
# Wait until all of the profiling information appears in the profile
# table.
timeout_seconds = 20
start_time = time.time()
while True:
profile_data = ray.timeline()
event_types = {event["cat"] for event in profile_data}
expected_types = [
"task",
"task:deserialize_arguments",
"task:execute",
"task:store_outputs",
"wait_for_function",
"ray.get",
"ray.put",
"ray.wait",
"submit_task",
"fetch_and_run_function",
# TODO (Alex) :https://github.com/ray-project/ray/pull/9346
# "register_remote_function",
"custom_event", # This is the custom one from ray.profile.
]
if all(expected_type in event_types
for expected_type in expected_types):
break
if time.time() - start_time > timeout_seconds:
raise RayTestTimeoutException(
"Timed out while waiting for information in "
"profile table. Missing events: {}.".format(
set(expected_types) - set(event_types)))
# The profiling information only flushes once every second.
time.sleep(1.1)
param_server = ParameterServer.remote()
# buffer
if use_per:
buffer = PrioritizedReplayBuffer.remote(size=buffer_max_size, alpha=priority_alpha)
else:
buffer = Buffer.remote(max_size=buffer_max_size)
# learner:
learner = Learner.remote(config, buffer, param_server, experiment_dir)
param_list = ray.get(learner.return_param_list.remote())
param_server.define_param_list.remote(param_list)
learner.update_param_server.remote()
# actors
actors = [Actor.remote(i, config, experiment_dir, buffer, param_server)
for i in range(n_agents)]
# processes
procs = []
procs.append(learner)
for actor in actors:
procs.append(actor)
print("run")
ray.wait([proc.run.remote() for proc in procs])
ray.timeline()
print("End")
def job_async(num_tasks, n):
task_result_ids = [task_async.remote() for task_id in range(num_tasks)]
task_results = ray.get(task_result_ids)
ray.timeline(filename="timeline.json")
return sum(task_results)
from tensorflow.keras.applications import ResNet50
from dkeras import dKeras
import numpy as np
import ray
ray.init()
data = np.random.uniform(-1, 1, (100, 224, 224, 3))
model = dKeras(ResNet50, init_ray=False, wait_for_workers=True, n_workers=4)
preds = model.predict(data)
ray.timeline('test')
"""
int8:
155.15053375276426
6.444483995437622 155.17146147122887
"""
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',filename='app.log',level=logging.DEBUG)
print(os.getpid())
print(os.getppid())
if not ray.is_initialized():
ray.init(include_webui=True)
files = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M']
db.setup()
print(os.getpid())
print(os.getppid())
with ray.profile('Event'):
for i in range(10):
time.sleep(randint(0, 4))
try:
ray.get(worker.remote(i))
except Exception as e:
raise e
print(e.message)
finally:
print('finally')
#results = [worker.remote(file) for file in (files)]
#ray.get(results)
if __name__ == '__main__':
main()
ray.timeline(filename='timeline.dump')
print(ray.errors())
do_tune=False,
do_sacred=False,
do_tqdm=True,
do_exit=False)
else:
kwargs = {'num_cpus': args.n_cpus}
if args.n_cpus == 0:
kwargs = {'num_cpus': 1, 'local_mode': True}
ray.init(**kwargs, num_gpus=args.n_gpus, include_dashboard=True)
if args.resume:
gin.bind_parameter('tune_run.resume', True)
if args.load_checkpoint:
gin.bind_parameter('Config.load_checkpoint', args.load_checkpoint)
gin.bind_parameter('Config.load_new_config', args.load_new_config)
if args.epochs_override is not None:
def set_train_steps():
gin.bind_parameter('Config.train_steps', args.epochs_override)
config.append(set_train_steps)
# TODO: remove heavy metrics s.a. weights in experiment_state.json
# gin.bind_parameter('tune_run.loggers', )
learner_gin_sacred(config, nofail=args.nofail)
if args.ray_debug_timeline:
ray.timeline(filename=args.ray_debug_timeline)
def export_timeline():
timestr = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
filename = f"/tmp/ray-timeline-{timestr}.json"
ray.timeline(filename=filename)
logging.info(f"Exported Ray timeline to {filename}")
write_rois(rass_data.output('v_%s.txt' % treatment_name, subfolder="%s" % treatment_name), totalDoses, roi_marks, kmax, jmax, imax, plan_grid_ct, v2Drow)
if options["debug_beam_doses"]:
vmc.saveToVTI(rass_data.output("mcp_doses"), mcDosesVMC, [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
if options["out_mc_doses"]:
vmc.saveToVTI(rass_data.output("mc_doses"), np.reshape(mcDoses, np.prod([imax, jmax, kmax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
if options["out_mc_doses_fluence"]:
vmc.saveToVTI(rass_data.output("mc_doses_fluence"), np.reshape(mcDosesFluence, np.prod([imax, jmax, kmax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
npTotalDoses = np.array(totalDoses, dtype=np.float32)
if options["out_difference"]:
vmc.saveToVTI(rass_data.output("npTotalDoses"), np.reshape(npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
vmc.saveToVTI(rass_data.output("difference"), np.reshape(mcDosesFluence-npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
if options["out_total_doses"]:
log.info("Max total npTotalDoses: %f" % np.max(npTotalDoses))
vmc.saveToVTI(rass_data.output("totalDoses"), np.reshape(npTotalDoses, np.prod([kmax, jmax, imax])), [dx, dy, dz], [kmax, jmax, imax], [xbase, ybase, zbase])
ray.timeline("/tmp/ray-timeline.json")
if options["histograms"]:
def generate_gnuplot_histograms(name, subfolder, doses_to_plot):
stats_fname = rass_data.output(f"statistics_{name}.txt");
png_fname = '%s_histogram_%s.png' % (treatment_name, name)
log.info(f"Generating histograms for {name}... (statistics will be saved to: {stats_fname})")
roi_statistics = {}
fout = open(stats_fname,'w')
f = open(rass_data.output('histograms.gpt', subfolder=subfolder), 'w')
f.write('set grid\nset style data lp\nset xlabel \'Dose [cGy]\'\n'
'set ylabel \'%% of volume\'\nset yrange [0:110]\nset term png size 1024,768\nset output "%s"\nplot ' % png_fname)
for r in range(len(myROIs)):
def main():
ray.init(address='10.155.1.10:6379', redis_password='******')
ray.timeline(filename="timeline.json")
start_time = time.time()
var_names = [
'beta', 'theta', 'p', 'sigma', 'rho', 'epsilonA', 'epsilonI',
'lambda0', 'gammaA', 'gammaI', 'gammaD', 'cD', 'dD', 'dI', 'delta',
'S0', 'R0', 'Q0', 'E0', 'A0', 'I0', 'D0', 'day'
]
paramp = pp.ParamProcessor.remote('memory-edo', var_names)
workload = dt.build_database(False)
workerlist = list()
cpu = 0
for node in ray.nodes():
cpu += int(node['Resources']['CPU'])
#filter_list = ['Goias', 'Sao_Paulo', 'Egypt', 'Ethiopia', 'France']
#filter_list = dt.brasilian_regions()
filter_list = workload.keys()
apply_filter(workload, filter_list)
print(f'Workload is {len(workload)}')
work_chunk = submit_work(workload, filter_list, workerlist, paramp,
program, cpu + 4)
print(
f'Submitting {work_chunk} tasks ({len(workload)}/{len(workerlist)} to go).'
)
check_size = 4
with open('log/processed.log', 'w') as f:
loops = 0
while len(workerlist) > 0:
num_returns = check_size if len(workerlist) >= check_size else len(
workerlist)
ready, not_ready = ray.wait(workerlist, num_returns=num_returns)
done = ray.get(ready)
printx(3, 0, f'Iteration : {loops}')
printx(4, 0, f'Ready length, regions : {len(ready)}')
printx(5, 0, f'Not Ready length : {len(not_ready)}')
printx(5, 0, f'Workload : {len(workload)}')
printx(4, 40, 'Ready List')
for pos, i in enumerate(done):
printx(5 + pos, 40, f'{i}')
elapsed_time = time.time() - start_time
stetime = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
printx(7, 0, f'Elapsed time : {stetime}')
for nct in done:
f.write(f'{nct}\n')
workerlist = not_ready
if len(workerlist) < cpu:
submit_work(workload, filter_list, workerlist, paramp, program,
cpu - len(workerlist))
f.flush()
time.sleep(15)
loops += 1
elapsed_time = time.time() - start_time
stetime = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print(f'Loops to process = {loops} in {stetime}.')
paramp.dump.remote('log/param.csv')
return
def apex(remote_learner,
remote_actors,
main_buffer,
remote_counter,
remote_evaluate,
log_wandb=False,
max_eps=1000,
replay_start_size=1000,
batch_size=128,
sync_nn_mod=100,
number_of_batchs=16,
beta=0.4,
rollout_size=70,
save_nn_mod=100,
save_dir='train'):
# Start tasks
online_weights, target_weights = remote_learner.get_weights.remote()
start_learner = False
rollouts = {}
for a in remote_actors:
rollouts[a.rollout.remote(rollout_size, online_weights,
target_weights)] = a
rollouts[remote_evaluate.test.remote(save_dir, online_weights,
target_weights)] = remote_evaluate
episodes_done = ray.get(remote_counter.get_value.remote())
optimization_step = 0
priority_dict, ds = None, None
# Main train process
while episodes_done < max_eps:
ready_ids, _ = ray.wait(list(rollouts))
first_id = ready_ids[0]
first = rollouts.pop(first_id)
if first == remote_learner:
optimization_step += 1
start_time = timeit.default_timer()
if optimization_step % sync_nn_mod == 0:
save = (optimization_step % save_nn_mod == 0) * save_dir
online_weights, target_weights = first.get_weights.remote(save)
rollouts[first.update_from_ds.remote(ds, start_time,
batch_size)] = first
indexes, priorities = ray.get(first_id)
indexes = indexes.copy()
priorities = priorities.copy()
main_buffer.update_priorities(indexes=indexes,
priorities=priorities)
ds = main_buffer.sample(number_of_batchs * batch_size, beta)
elif first == remote_evaluate:
score, eps_number = ray.get(first_id)
if log_wandb:
wandb.log({'Score': score, 'episode': eps_number})
rollouts[remote_evaluate.test.remote(
save_dir, online_weights, target_weights)] = remote_evaluate
else:
rollouts[first.rollout.remote(rollout_size, online_weights,
target_weights)] = first
data, priorities = ray.get(first_id)
priorities = priorities.copy()
main_buffer.add(priorities=priorities, **data)
if main_buffer.get_stored_size(
) > replay_start_size and not start_learner:
start_time = timeit.default_timer()
ds = main_buffer.sample(number_of_batchs * batch_size, beta)
rollouts[remote_learner.update_from_ds.remote(
ds, start_time, batch_size)] = remote_learner
ds = main_buffer.sample(number_of_batchs * batch_size, beta)
start_learner = True
episodes_done = ray.get(remote_counter.get_value.remote())
ray.timeline()
simulator = Simulator(MODEL_GLCWB)
results = {}
for f_simulate in [po_bolus, iv_bolus, iv_infusion, clamp, mix, stepped_clamp]:
f_key = f_simulate.__name__
print(f_key)
result = f_simulate(simulator, r)
print(result)
print("results:", len(result), sys.getsizeof(result))
results[f_key] = result
for key, result in results.items():
somatostatin_plot(result, title=key)
plt.show()
import ray
ray.timeline(filename="timeline.json")
'''
def exlude_pkdb_parameter_filter(pid):
""" Returns True if excluded, False otherwise
:param pid:
:return:
"""
# TODO: implement
# dose parameters
if (pid.startswith("IVDOSE_")) or (pid.startswith("PODOSE_")):
return True
# physical parameters
if (pid.startswith("Mr_")) or pid in ["R_PDB"]:
def main():
#pype.init_ray()
ray.init()
ray.timeline(filename="timeline.json")
server = pype.Server.remote()
server.add.remote('frame', use_locking=False)
#server.add.remote('person', use_locking=False)
# # server.add.remote('face', use_locking=False)
server.add.remote('pose', use_locking=False)
server.add.remote('object', use_locking=False)
server.add.remote('action', use_locking=False)
# server.add.remote('action', use_locking=False)
# server.add.remote('action1', use_locking=False)
# server.add.remote('counter', use_locking=False)
# server.add.remote('results')
person_models = [
NullDetector.remote(server,
input_queue='frame',
output_queues=['pose'],
batch_size=-1) for _ in range(1)
]
person_models = [
NullDetector.remote(server,
input_queue='pose',
output_queues=['object'],
batch_size=-1) for _ in range(1)
]
person_models = [
NullDetector.remote(server,
input_queue='object',
output_queues=['action'],
batch_size=-1) for _ in range(1)
]
# person_models = [NullDetector.remote(server, input_queue='action',
# output_queues=['action1'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action1',
# output_queues=['action2'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action2',
# output_queues=['action3'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action3',
# output_queues=['action4'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action4',
# output_queues=['action5'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action5',
# output_queues=['action6'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action6',
# output_queues=['action7'],
# batch_size=-1) for _ in range(1)]
time.sleep(3)
ray.timeline(filename="timeline.json")
for _ in range(1):
NullVideoServer.remote(server, output_queues='frame')
# person_models = [NullDetector.remote(server, input_queue='action1',
# output_queues=['action2'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action2',
# output_queues=['action3'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action3',
# output_queues=['action4'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action4',
# output_queues=['action5'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action5',
# output_queues=['action6'],
# batch_size=-1) for _ in range(1)]
# person_models = [NullDetector.remote(server, input_queue='action6',
# output_queues=['action7'],
# batch_size=-1) for _ in range(1)]
# pose_models = [NullDetector.remote(server, input_queue='pose',
# output_queues=['object']) for _ in range(1)]
# face_models = [NullDetector.remote(server, input_queue='face',
# output_queues=['results']) for _ in range(1)]
# counter_models = [NullDetector.remote(server, input_queue='person',
# output_queues=['results']) for _ in range(1)]
# object_models = [NullDetector.remote(server, input_queue='object',
# output_queues=['action']) for _ in range(1)]
# action_models = [NullDetector.remote(server, input_queue='action',
# output_queues=['results']) for _ in range(1)]
ray.get(server.all_initalized.remote())
time_elapsed = 0
count = 0
start_time = time.time()
ray.timeline(filename="timeline.json")
while True:
pype.pull_wait(server, 'action')
data = ray.get(server.pull.remote('action'))
#time_elapsed += time.time()-data[0]
count += len(data)
# print(count)
if time.time() - start_time > 10:
ray.timeline(filename="timeline.json")
break
print("Throughput: {}".format(count / (time.time() - start_time)))
ray.shutdown()
def get(self):
traceJson = ray.timeline()
return traceJson
def chrome_tracing_dump(self, filename=None):
logger.warning(
"ray.global_state.chrome_tracing_dump() is deprecated and will be "
"removed in a subsequent release. Use ray.timeline() instead.")
return ray.timeline(filename=filename)
nargs=1,
default=None,
help=
"directory to store the encrypted ballots on disk, enables equivalence checking (default: memory only)",
)
parser.add_argument(
"cvr_file",
type=str,
nargs="+",
help="filename(s) for the Dominion-style ballot CVR file",
)
args = parser.parse_args()
files = args.cvr_file
file_dir = args.dir[0] if args.dir else None
use_progressbar = args.progress
if args.local:
print("Using Ray locally")
ray_init_localhost()
else:
print("Using Ray on a cluster")
ray_init_cluster()
for arg in files:
run_bench(arg, file_dir, use_progressbar)
print("Writing Ray timelines to disk.")
ray.timeline("ray-timeline.json")
ray.object_transfer_timeline("ray-object-transfer-timeline.json")
