def __init__(self, scheme=None):
self.proc = None
self.daemon = None
self.stop = Event()
self.scheme = scheme
_fields_ = [
('length', ctypes.c_uint),
('data', ctypes.c_byte * 64)
]
def from_python(self, data):
length = len(data)
ctypes.memmove(self.data, (ctypes.c_byte * length).from_buffer(bytearray(data)), ctypes.sizeof(self.data))
self.length = length
def to_python(self):
return str(bytearray(self.data[:self.length]))
size = 100000
q = Queue(Data, size, True) # True makes the queue synchronized
run = Event()
run.set()
def worker(q, r):
while r.is_set():
try:
v = q.get()
assert(v.startswith('test'))
sleep(0.00001)
except Empty:
pass
def main(q):
v = 0
t = time()
while v != (size * 3):
from multiprocessing import Event e = Event() # e.set() # e.clear() # e.wait() # e.is_set() # 事件是通过is_set()的bool值，去标识e.wait() 的阻塞状态 # 当is_set()的bool值为False时，e.wait()是阻塞状态 # 当is_set()的bool值为True时，e.wait()是非阻塞状态 # 当使用set()时，是把is_set的bool变为True # 当使用clear()时，是把is_set的bool变为False print(e.is_set()) # False wait应该是阻塞住 e.set() # 将is_set 的bool值变为True，将wait变为非阻塞 e.wait() print(e.is_set()) print(123) e.clear() print(e.is_set()) e.wait() print(123)
def run(args):
# --------------------> Preparation <-------------------- #
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.set_default_dtype(torch.float64)
torch.set_num_threads(1)
device = torch.device("cpu")
# build a scalarization template
scalarization_template = WeightedSumScalarization(
num_objs=args.obj_num, weights=np.ones(args.obj_num) / args.obj_num)
total_num_updates = int(
args.num_env_steps) // args.num_steps // args.num_processes
start_time = time.time()
# initialize ep and population and opt_graph
ep = EP()
if args.obj_num == 2:
population = Population2d(args)
elif args.obj_num > 2:
population = Population3d(args)
else:
raise NotImplementedError
opt_graph = OptGraph()
# Construct tasks for warm up
elite_batch, scalarization_batch = initialize_warm_up_batch(args, device)
rl_num_updates = args.warmup_iter
for sample, scalarization in zip(elite_batch, scalarization_batch):
sample.optgraph_id = opt_graph.insert(deepcopy(scalarization.weights),
deepcopy(sample.objs), -1)
episode = 0
iteration = 0
while iteration < total_num_updates:
if episode == 0:
print_info(
'\n------------------------------- Warm-up Stage -------------------------------'
)
else:
print_info(
'\n-------------------- Evolutionary Stage: Generation {:3} --------------------'
.format(episode))
episode += 1
offspring_batch = np.array([])
# --------------------> RL Optimization <-------------------- #
# compose task for each elite
task_batch = []
for elite, scalarization in \
zip(elite_batch, scalarization_batch):
task_batch.append(Task(
elite, scalarization)) # each task is a (policy, weight) pair
# run MOPG for each task in parallel
processes = []
results_queue = Queue()
done_event = Event()
for task_id, task in enumerate(task_batch):
p = Process(target = MOPG_worker, \
args = (args, task_id, task, device, iteration, rl_num_updates, start_time, results_queue, done_event))
p.start()
processes.append(p)
# collect MOPG results for offsprings and insert objs into objs buffer
all_offspring_batch = [[] for _ in range(len(processes))]
cnt_done_workers = 0
while cnt_done_workers < len(processes):
rl_results = results_queue.get()
task_id, offsprings = rl_results['task_id'], rl_results[
'offspring_batch']
for sample in offsprings:
all_offspring_batch[task_id].append(Sample.copy_from(sample))
if rl_results['done']:
cnt_done_workers += 1
# put all intermidiate policies into all_sample_batch for EP update
all_sample_batch = []
# store the last policy for each optimization weight for RA
last_offspring_batch = [None] * len(processes)
# only the policies with iteration % update_iter = 0 are inserted into offspring_batch for population update
# after warm-up stage, it's equivalent to the last_offspring_batch
offspring_batch = []
for task_id in range(len(processes)):
offsprings = all_offspring_batch[task_id]
prev_node_id = task_batch[task_id].sample.optgraph_id
opt_weights = deepcopy(
task_batch[task_id].scalarization.weights).detach().numpy()
for i, sample in enumerate(offsprings):
all_sample_batch.append(sample)
if (i + 1) % args.update_iter == 0:
prev_node_id = opt_graph.insert(opt_weights,
deepcopy(sample.objs),
prev_node_id)
sample.optgraph_id = prev_node_id
offspring_batch.append(sample)
last_offspring_batch[task_id] = offsprings[-1]
done_event.set()
# -----------------------> Update EP <----------------------- #
# update EP and population
ep.update(all_sample_batch)
population.update(offspring_batch)
# ------------------- > Task Selection <--------------------- #
if args.selection_method == 'moead':
elite_batch, scalarization_batch = [], []
weights_batch = []
generate_weights_batch_dfs(0, args.obj_num, args.min_weight,
args.max_weight, args.delta_weight, [],
weights_batch)
for weights in weights_batch:
scalarization = deepcopy(scalarization_template)
scalarization.update_weights(weights)
scalarization_batch.append(scalarization)
best_sample, best_value = None, -np.inf
for sample in population.sample_batch:
value = scalarization.evaluate(torch.Tensor(sample.objs))
if value > best_value:
best_sample, best_value = sample, value
elite_batch.append(best_sample)
elif args.selection_method == 'prediction-guided':
elite_batch, scalarization_batch, predicted_offspring_objs = population.prediction_guided_selection(
args, iteration, ep, opt_graph, scalarization_template)
elif args.selection_method == 'random':
elite_batch, scalarization_batch = population.random_selection(
args, scalarization_template)
elif args.selection_method == 'ra':
elite_batch = last_offspring_batch
scalarization_batch = []
weights_batch = []
generate_weights_batch_dfs(0, args.obj_num, args.min_weight,
args.max_weight, args.delta_weight, [],
weights_batch)
for weights in weights_batch:
scalarization = deepcopy(scalarization_template)
scalarization.update_weights(weights)
scalarization_batch.append(scalarization)
elif args.selection_method == 'pfa':
if args.obj_num > 2:
raise NotImplementedError
elite_batch = last_offspring_batch
scalarization_batch = []
delta_ratio = (iteration + rl_num_updates + args.update_iter -
args.warmup_iter) / (total_num_updates -
args.warmup_iter)
delta_ratio = np.clip(delta_ratio, 0.0, 1.0)
for i in np.arange(args.min_weight,
args.max_weight + 0.5 * args.delta_weight,
args.delta_weight):
w = np.clip(i + delta_ratio * args.delta_weight,
args.min_weight, args.max_weight)
weights = np.array([abs(w), abs(1.0 - w)])
scalarization = deepcopy(scalarization_template)
scalarization.update_weights(weights)
scalarization_batch.append(scalarization)
else:
raise NotImplementedError
print_info('Selected Tasks:')
for i in range(len(elite_batch)):
print_info('objs = {}, weight = {}'.format(
elite_batch[i].objs, scalarization_batch[i].weights))
iteration = min(iteration + rl_num_updates, total_num_updates)
rl_num_updates = args.update_iter
# ----------------------> Save Results <---------------------- #
# save ep
ep_dir = os.path.join(args.save_dir, str(iteration), 'ep')
os.makedirs(ep_dir, exist_ok=True)
with open(os.path.join(ep_dir, 'objs.txt'), 'w') as fp:
for obj in ep.obj_batch:
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*obj))
# save population
population_dir = os.path.join(args.save_dir, str(iteration),
'population')
os.makedirs(population_dir, exist_ok=True)
with open(os.path.join(population_dir, 'objs.txt'), 'w') as fp:
for sample in population.sample_batch:
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(sample.objs)))
# save optgraph and node id for each sample in population
with open(os.path.join(population_dir, 'optgraph.txt'), 'w') as fp:
fp.write('{}\n'.format(len(opt_graph.objs)))
for i in range(len(opt_graph.objs)):
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' + ';{:5f}' +
(args.obj_num - 1) * ',{:5f}' + ';{}\n').format(
*(opt_graph.weights[i]), *(opt_graph.objs[i]),
opt_graph.prev[i]))
fp.write('{}\n'.format(len(population.sample_batch)))
for sample in population.sample_batch:
fp.write('{}\n'.format(sample.optgraph_id))
# save elites
elite_dir = os.path.join(args.save_dir, str(iteration), 'elites')
os.makedirs(elite_dir, exist_ok=True)
with open(os.path.join(elite_dir, 'elites.txt'), 'w') as fp:
for elite in elite_batch:
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(elite.objs)))
with open(os.path.join(elite_dir, 'weights.txt'), 'w') as fp:
for scalarization in scalarization_batch:
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(scalarization.weights)))
if args.selection_method == 'prediction-guided':
with open(os.path.join(elite_dir, 'predictions.txt'), 'w') as fp:
for objs in predicted_offspring_objs:
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(objs)))
with open(os.path.join(elite_dir, 'offsprings.txt'), 'w') as fp:
for i in range(len(all_offspring_batch)):
for j in range(len(all_offspring_batch[i])):
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(all_offspring_batch[i][j].objs)))
# ----------------------> Save Final Model <----------------------
os.makedirs(os.path.join(args.save_dir, 'final'), exist_ok=True)
# save ep policies & env_params
for i, sample in enumerate(ep.sample_batch):
torch.save(
sample.actor_critic.state_dict(),
os.path.join(args.save_dir, 'final', 'EP_policy_{}.pt'.format(i)))
with open(
os.path.join(args.save_dir, 'final',
'EP_env_params_{}.pkl'.format(i)), 'wb') as fp:
pickle.dump(sample.env_params, fp)
# save all ep objectives
with open(os.path.join(args.save_dir, 'final', 'objs.txt'), 'w') as fp:
for i, obj in enumerate(ep.obj_batch):
fp.write(('{:5f}' + (args.obj_num - 1) * ',{:5f}' +
'\n').format(*(obj)))
# save all ep env_params
if args.obj_rms:
with open(os.path.join(args.save_dir, 'final', 'env_params.txt'),
'w') as fp:
for sample in ep.sample_batch:
fp.write('obj_rms: mean: {} var: {}\n'.format(
sample.env_params['obj_rms'].mean,
sample.env_params['obj_rms'].var))
def state_lattice_planner(n: int, m: int, file_name: str = "test", g_weight: float = 0.5, h_weight: float = 0.5,
costmap_file: str = "",
start_pos: tuple = (20, 10, 0), goal_pos: tuple = (20, 280, 0),
initial_heading: float = math.pi / 2, padding: int = 0,
turning_radius: int = 8, vel: int = 10, num_headings: int = 8,
num_obs: int = 130, min_r: int = 1, max_r: int = 8, upper_offset: int = 20,
lower_offset: int = 20, allow_overlap: bool = False,
obstacle_density: int = 6, obstacle_penalty: float = 3,
Kp: float = 3, Ki: float = 0.08, Kd: float = 0.5, inf_stream: bool = False,
save_animation: bool = False, save_costmap: bool = False, smooth_path: bool = False,
replan: bool = False, horizon: int = np.inf, y_axis_limit: int = 100, buffer: int = None,
move_yaxis_threshold: int = 20, new_obs_dist: int = None):
# PARAM SETUP
# --- costmap --- #
load_costmap_file = costmap_file
ship_vertices = np.array([[-1, -4],
[1, -4],
[1, 2],
[0, 4],
[-1, 2]])
# load costmap object from file if specified
if load_costmap_file:
with open(load_costmap_file, "rb") as fd:
costmap_obj = pickle.load(fd)
# recompute costmap costs if obstacle penalty is different than original
if costmap_obj.obstacle_penalty != obstacle_penalty:
costmap_obj.update2(obstacle_penalty)
else:
# initialize costmap
costmap_obj = CostMap(
n, m, obstacle_penalty, min_r, max_r, inf_stream, y_axis_limit, num_obs, new_obs_dist
)
# generate obs up until buffer if in inf stream mode
max_y = y_axis_limit + buffer if inf_stream else goal_pos[1]
# generate random obstacles
costmap_obj.generate_obstacles(start_pos[1], max_y, num_obs,
upper_offset, lower_offset, allow_overlap)
orig_obstacles = costmap_obj.obstacles.copy()
# initialize ship object
ship = Ship(ship_vertices, start_pos, initial_heading, turning_radius, padding)
# get the primitives
prim = Primitives(turning_radius, initial_heading, num_headings)
# generate swath dict
swath_dict = swath.generate_swath(ship, prim)
print("WEIGHTS", g_weight, h_weight)
# initialize a star object
a_star = AStar(g_weight, h_weight, cmap=costmap_obj,
primitives=prim, ship=ship, first_initial_heading=initial_heading)
# compute current goal
curr_goal = (goal_pos[0], min(goal_pos[1], (start_pos[1] + horizon)), goal_pos[2])
t0 = time.time()
worked, smoothed_edge_path, nodes_visited, x1, y1, x2, y2, orig_path = \
a_star.search(start_pos, curr_goal, swath_dict, smooth_path)
init_plan_time = time.time() - t0
print("Time elapsed: ", init_plan_time)
print("Hz", 1 / init_plan_time)
if worked:
plot_obj = Plot(
costmap_obj, prim, ship, nodes_visited, smoothed_edge_path.copy(),
path_nodes=(x1, y1), smoothing_nodes=(x2, y2), horizon=horizon,
inf_stream=inf_stream, y_axis_limit=y_axis_limit
)
path = Path(plot_obj.full_path)
else:
print("Failed to find path at step 0")
exit(1)
# init pymunk sim
space = pymunk.Space()
space.add(ship.body, ship.shape)
space.gravity = (0, 0)
staticBody = space.static_body # create a static body for friction constraints
# create the pymunk objects and the polygon patches for the ice
polygons = [
create_polygon(
space, staticBody, (obs['vertices'] - np.array(obs['centre'])).tolist(),
*obs['centre'], density=obstacle_density
)
for obs in costmap_obj.obstacles
]
# From pure pursuit
state = State(x=start_pos[0], y=start_pos[1], yaw=0.0, v=0.0)
target_course = TargetCourse(path.path[0], path.path[1])
target_ind = target_course.search_target_index(state)
# init PID controller
pid = PID(Kp, Ki, Kd, 0)
pid.output_limits = (-1, 1) # limit on PID output
# generator to end matplotlib animation when it reaches the goal
def gen():
nonlocal at_goal
i = 0
while not at_goal:
i += 1
yield i
raise StopIteration # should stop animation
def animate(frame, queue_state, pipe_path):
nonlocal at_goal
steps = 10
# move simulation forward 20 ms seconds:
for x in range(steps):
space.step(0.02 / steps)
# get current state
ship_pos = (ship.body.position.x, ship.body.position.y, 0) # straight ahead of boat is 0
# check if ship has made it past the goal line
if ship.body.position.y >= goal_pos[1]:
at_goal = True
print("\nAt goal, shutting down...")
plt.close(plot_obj.map_fig)
plt.close(plot_obj.sim_fig)
queue_state.close()
shutdown_event.set()
return []
# Pymunk takes left turn as negative and right turn as positive in ship.body.angle
# To get proper error, we must flip the sign on the angle, as to calculate the setpoint,
# we look at a point one lookahead distance ahead, and find the angle to that point with
# arctan2, but using this, we will get positive values on the left and negative values on the right
# As the angular velocity in pymunk uses the same convention as ship.body.angle, we must flip the sign
# of the output as well
output = -pid(-ship.body.angle)
# should play around with frequency at which new state data is sent
if frame % 20 == 0 and frame != 0 and replan:
# update costmap and polygons
to_add, to_remove = costmap_obj.update(polygons, ship.body.position.y)
assert len(costmap_obj.obstacles) <= costmap_obj.total_obs
# remove polygons if any
for obs in to_remove:
polygons.remove(obs)
# add polygons if any
polygons.extend([
create_polygon(
space, staticBody, (obs['vertices'] - np.array(obs['centre'])).tolist(),
*obs['centre'], density=obstacle_density
)
for obs in to_add
])
print("Total polygons", len(polygons))
try:
# empty queue to ensure latest state data is pushed
queue_state.get_nowait()
except Empty:
pass
# send updated state via queue
queue_state.put({
'ship_pos': ship_pos,
'ship_body_angle': ship.body.angle,
'costmap': costmap_obj.cost_map,
'obstacles': costmap_obj.obstacles,
}, block=False)
print('\nSent new state data!')
# check if there is a new path
if pipe_path.poll():
# get new path
path_data = pipe_path.recv()
new_path = path_data['path'] # this is the full path and in the correct order i.e. start -> goal
print('\nReceived replanned path!')
# compute swath cost of new path up until the max y distance of old path for a fair comparison
# note, we do not include the path length in the cost
full_swath, full_cost, current_cost = swath.compute_swath_cost(
costmap_obj.cost_map, new_path, ship.vertices, threshold_dist=path.path[1][-1]
)
try:
assert full_cost >= current_cost # sanity check
except AssertionError:
print("Full and partial swath costs", full_cost, current_cost)
path_expired = False
prev_cost = None
# check if old path is 'expired' regardless of costs
if (path.path[1][-1] - ship_pos[1]) < horizon / 2:
path_expired = True
else:
# clip old path based on ship y position
old_path = path.clip_path(ship_pos[1])
# compute cost of clipped old path
_, prev_cost, _ = swath.compute_swath_cost(costmap_obj.cost_map, old_path, ship.vertices)
print('\nPrevious Cost: {prev_cost:.3f}'.format(prev_cost=prev_cost))
print('Current Cost: {current_cost:.3f}\n'.format(current_cost=current_cost))
if path_expired or current_cost < prev_cost:
if path_expired:
print("Path expired, applying new path regardless of cost!")
else:
print("New path better than old path!")
path.new_path_cnt += 1
plot_obj.update_path(
new_path, full_swath, path_data['path_nodes'],
path_data['smoothing_nodes'], path_data['nodes_expanded']
)
# update to new path
path.path = new_path
ship.set_path_pos(0)
# update pure pursuit objects with new path
target_course.update(path.path[0], path.path[1])
state.update(ship.body.position.x, ship.body.position.y, ship.body.angle)
# update costmap and map fig
plot_obj.update_map(costmap_obj.cost_map)
plot_obj.map_fig.canvas.draw()
else:
print("Old path better than new path")
path.old_path_cnt += 1
if ship.path_pos < np.shape(path.path)[1] - 1:
# Translate linear velocity into direction of ship
x_vel = math.sin(ship.body.angle)
y_vel = math.cos(ship.body.angle)
mag = math.sqrt(x_vel ** 2 + y_vel ** 2)
x_vel = x_vel / mag * vel
y_vel = y_vel / mag * vel
ship.body.velocity = Vec2d(x_vel, y_vel)
# Assign output of PID controller to angular velocity
ship.body.angular_velocity = output
# Update the pure pursuit state
state.update(ship.body.position.x, ship.body.position.y, ship.body.angle)
# Get look ahead index
ind = target_course.search_target_index(state)
if ind != ship.path_pos:
# Find heading from current position to look ahead point
ship.set_path_pos(ind)
dy = path.path[1][ind] - ship.body.position.y
dx = path.path[0][ind] - ship.body.position.x
angle = np.arctan2(dy, dx) - a_star.first_initial_heading
# set setpoint for PID controller
pid.setpoint = angle
# at each step animate ship and obstacle patches
plot_obj.animate_ship(ship, horizon, move_yaxis_threshold)
plot_obj.animate_obstacles(polygons)
return plot_obj.get_sim_artists()
# multiprocessing setup
lifo_queue = Queue(maxsize=1) # LIFO queue to send state information to A*
conn_recv, conn_send = Pipe(duplex=False) # pipe to send new path to controller and for plotting
shutdown_event = Event()
# setup a process to run A*
print('\nStart process...')
gen_path_process = Process(
target=gen_path, args=(lifo_queue, conn_send, shutdown_event, ship, prim,
costmap_obj, swath_dict, a_star, goal_pos, horizon, smooth_path)
)
gen_path_process.start()
# init vars used in animation methods
at_goal = False
# start animation in main process
anim = animation.FuncAnimation(plot_obj.sim_fig,
animate,
frames=gen,
fargs=(lifo_queue, conn_recv,),
interval=20,
blit=False,
repeat=False,
)
if save_animation:
anim.save(os.path.join('gifs', file_name), writer=animation.PillowWriter(fps=30))
plt.show()
total_dist_moved = 0
# Compare cost maps
for i, j in zip(orig_obstacles, polygons):
pos = (j.body.position.x, j.body.position.y)
area = j.area
if area > 4:
total_dist_moved = a_star.dist(i['centre'], pos) * (area/2) + total_dist_moved
print('TOTAL DIST MOVED', total_dist_moved)
print('Old/new path counts', '\n\told path', path.old_path_cnt, '\n\tnew path', path.new_path_cnt)
shutdown_event.set()
print('\n...done with process')
gen_path_process.join()
print('Completed multiprocessing')
# get response from user for saving costmap
if save_costmap:
costmap_obj.save_to_disk()
return total_dist_moved, init_plan_time
def test_last_blob_retrieval(self):
kill_event = Event()
dead_event_1 = Event()
blob_hash_queue_1 = Queue()
blob_hash_queue_2 = Queue()
fast_uploader = Process(target=start_blob_uploader,
args=(blob_hash_queue_1, kill_event,
dead_event_1, False))
fast_uploader.start()
self.server_processes.append(fast_uploader)
dead_event_2 = Event()
slow_uploader = Process(target=start_blob_uploader,
args=(blob_hash_queue_2, kill_event,
dead_event_2, True))
slow_uploader.start()
self.server_processes.append(slow_uploader)
logging.debug("Testing transfer")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 2)
hash_announcer = FakeAnnouncer()
rate_limiter = DummyRateLimiter()
db_dir, blob_dir = mk_db_and_blob_dir()
self.session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=5553,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1],
external_ip="127.0.0.1")
d1 = self.wait_for_hash_from_queue(blob_hash_queue_1)
d2 = self.wait_for_hash_from_queue(blob_hash_queue_2)
d = defer.DeferredList([d1, d2], fireOnOneErrback=True)
def get_blob_hash(results):
self.assertEqual(results[0][1], results[1][1])
return results[0][1]
d.addCallback(get_blob_hash)
def download_blob(blob_hash):
prm = self.session.payment_rate_manager
downloader = StandaloneBlobDownloader(blob_hash,
self.session.blob_manager,
peer_finder, rate_limiter,
prm, wallet)
d = downloader.download()
return d
def start_transfer(blob_hash):
logging.debug("Starting the transfer")
d = self.session.setup()
d.addCallback(lambda _: download_blob(blob_hash))
return d
d.addCallback(start_transfer)
def stop(arg):
if isinstance(arg, Failure):
logging.debug("Client is stopping due to an error. Error: %s",
arg.getTraceback())
else:
logging.debug("Client is stopping normally.")
kill_event.set()
logging.debug("Set the kill event")
d1 = self.wait_for_event(dead_event_1, 15)
d2 = self.wait_for_event(dead_event_2, 15)
dl = defer.DeferredList([d1, d2])
def print_shutting_down():
logging.info("Client is shutting down")
dl.addCallback(lambda _: print_shutting_down())
dl.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir))
dl.addCallback(lambda _: arg)
return dl
d.addBoth(stop)
return d
def test_multiple_uploaders(self):
sd_hash_queue = Queue()
num_uploaders = 3
kill_event = Event()
dead_events = [Event() for _ in range(num_uploaders)]
ready_events = [Event() for _ in range(1, num_uploaders)]
lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_events[0],
5209343, 9373419, 2**22)
uploader = Process(target=lbry_uploader.start)
uploader.start()
self.server_processes.append(uploader)
logging.debug("Testing multiple uploaders")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, num_uploaders)
hash_announcer = FakeAnnouncer()
rate_limiter = DummyRateLimiter()
sd_identifier = StreamDescriptorIdentifier()
db_dir, blob_dir = mk_db_and_blob_dir()
self.session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=5553,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1],
external_ip="127.0.0.1")
self.stream_info_manager = TempEncryptedFileMetadataManager()
self.lbry_file_manager = EncryptedFileManager(self.session,
self.stream_info_manager,
sd_identifier)
def start_additional_uploaders(sd_hash):
for i in range(1, num_uploaders):
uploader = Process(target=start_lbry_reuploader,
args=(sd_hash, kill_event, dead_events[i],
ready_events[i - 1], i, 2**10))
uploader.start()
self.server_processes.append(uploader)
return defer.succeed(True)
def wait_for_ready_events():
return defer.DeferredList([
self.wait_for_event(ready_event, 60)
for ready_event in ready_events
])
def make_downloader(metadata, prm):
info_validator = metadata.validator
options = metadata.options
factories = metadata.factories
chosen_options = [
o.default_value
for o in options.get_downloader_options(info_validator, prm)
]
return factories[0].make_downloader(metadata, chosen_options, prm)
def download_file(sd_hash):
prm = self.session.payment_rate_manager
d = download_sd_blob(self.session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm)
d.addCallback(lambda downloader: downloader.start())
return d
def check_md5_sum():
f = open('test_file')
hashsum = MD5.new()
hashsum.update(f.read())
self.assertEqual(hashsum.hexdigest(),
"e5941d615f53312fd66638239c1f90d5")
def start_transfer(sd_hash):
logging.debug("Starting the transfer")
d = start_additional_uploaders(sd_hash)
d.addCallback(lambda _: wait_for_ready_events())
d.addCallback(lambda _: self.session.setup())
d.addCallback(
lambda _: add_lbry_file_to_sd_identifier(sd_identifier))
d.addCallback(lambda _: self.lbry_file_manager.setup())
d.addCallback(lambda _: download_file(sd_hash))
d.addCallback(lambda _: check_md5_sum())
return d
def stop(arg):
if isinstance(arg, Failure):
logging.debug("Client is stopping due to an error. Error: %s",
arg.getTraceback())
else:
logging.debug("Client is stopping normally.")
kill_event.set()
logging.debug("Set the kill event")
d = defer.DeferredList([
self.wait_for_event(dead_event, 15)
for dead_event in dead_events
])
def print_shutting_down():
logging.info("Client is shutting down")
d.addCallback(lambda _: print_shutting_down())
d.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir))
d.addCallback(lambda _: arg)
return d
d = self.wait_for_hash_from_queue(sd_hash_queue)
d.addCallback(start_transfer)
d.addBoth(stop)
return d
if args.username and args.password:
es = Elasticsearch(url.netloc,
request_timeout=5,
timeout=args.timeout,
http_auth=(args.username, args.password))
if url.scheme == 'https':
es = Elasticsearch(url.netloc,
use_ssl=True,
verify_certs=False,
request_timeout=5,
timeout=args.timeout,
http_auth=(args.username, args.password))
else:
es = Elasticsearch(url.netloc, request_timeout=5, timeout=args.timeout)
if url.scheme == 'https':
es = Elasticsearch(url.netloc,
use_ssl=True,
verify_certs=False,
request_timeout=5,
timeout=args.timeout)
outq = Queue(maxsize=50000)
alldone = Event()
dumpproc = Process(target=dump, args=(es, outq, alldone))
dumpproc.daemon = True
dumpproc.start()
while not alldone.is_set() or outq.qsize() > 0:
try:
print json.dumps(outq.get(block=False))
except:
time.sleep(0.1)
def __init__(self, *args, **kwargs):
self._connection = None
self.parent_id = kwargs.pop('parent_id')
super(BaseWorker, self).__init__(*args, **kwargs)
self.should_exit = Event()
def test_lbry_transfer(self):
sd_hash_queue = Queue()
kill_event = Event()
dead_event = Event()
lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_event,
5209343)
uploader = Process(target=lbry_uploader.start)
uploader.start()
self.server_processes.append(uploader)
logging.debug("Testing transfer")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = DummyRateLimiter()
sd_identifier = StreamDescriptorIdentifier()
db_dir, blob_dir = mk_db_and_blob_dir()
self.session = Session(conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=5553,
dht_node_port=4445,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
dht_node_class=FakeNode,
is_generous=self.is_generous,
external_ip="127.0.0.1")
self.lbry_file_manager = EncryptedFileManager(self.session,
sd_identifier)
def make_downloader(metadata, prm):
factories = metadata.factories
return factories[0].make_downloader(metadata,
prm.min_blob_data_payment_rate,
prm, db_dir)
def download_file(sd_hash):
prm = self.session.payment_rate_manager
d = download_sd_blob(self.session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm)
d.addCallback(lambda downloader: downloader.start())
return d
def check_md5_sum():
f = open(os.path.join(db_dir, 'test_file'))
hashsum = MD5.new()
hashsum.update(f.read())
self.assertEqual(hashsum.hexdigest(),
"4ca2aafb4101c1e42235aad24fbb83be")
@defer.inlineCallbacks
def start_transfer(sd_hash):
logging.debug("Starting the transfer")
yield self.session.setup()
yield add_lbry_file_to_sd_identifier(sd_identifier)
yield self.lbry_file_manager.setup()
yield download_file(sd_hash)
yield check_md5_sum()
def stop(arg):
if isinstance(arg, Failure):
logging.debug("Client is stopping due to an error. Error: %s",
arg.getTraceback())
else:
logging.debug("Client is stopping normally.")
kill_event.set()
logging.debug("Set the kill event")
d = self.wait_for_event(dead_event, 15)
def print_shutting_down():
logging.info("Client is shutting down")
d.addCallback(lambda _: print_shutting_down())
d.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir))
d.addCallback(lambda _: arg)
return d
d = self.wait_for_hash_from_queue(sd_hash_queue)
d.addCallback(start_transfer)
d.addBoth(stop)
return d
def __init__(self):
super(RegistryServer, self).__init__()
self.notify = Event()
def __init__(self, args):
super().__init__()
self.args = args
self.is_ready = Event()
def _run_tests(self, tests, **kwargs):
# tests = dict where the key is a test group name and the value are
# the tests to run
tests_queue = Queue()
results_queue = Queue()
stop_event = Event()
pending_tests = {}
pending_not_thread_safe_tests = {}
completed_tests = {}
failures = 0
errors = 0
start_time = time.time()
# First tun tests which are not thread safe in the main process
for group in self._not_thread_safe:
if group not in tests.keys():
continue
group_tests = tests[group]
del tests[group]
logger.debug('Running tests in a main process: %s' % (group_tests))
pending_not_thread_safe_tests[group] = group_tests
result = self._tests_func(tests=group_tests, worker_index=None)
results_queue.put((group, result), block=False)
for group, tests in tests.items():
tests_queue.put((group, tests), block=False)
pending_tests[group] = tests
worker_count = self._worker_count
if worker_count == 'auto':
worker_count = len(pending_tests)
elif worker_count == 'cpu':
worker_count = multiprocessing.cpu_count()
if worker_count > len(pending_tests):
# No need to spawn more workers then there are tests.
worker_count = len(pending_tests)
worker_max = self._worker_max
if worker_max == 'auto':
worker_max = len(pending_tests)
elif worker_max == 'cpu':
worker_max = multiprocessing.cpu_count()
if worker_count > worker_max:
# No need to spawn more workers then there are tests.
worker_count = worker_max
worker_args = (tests_queue, results_queue, stop_event)
logger.debug("Number of workers %s " % worker_count)
workers = self._create_worker_pool(pool_size=worker_count,
target_func=self._run_tests_worker,
worker_args=worker_args)
for index, worker in enumerate(workers):
logger.debug('Staring worker %s' % (index))
worker.start()
if workers:
while pending_tests:
try:
try:
group, result = results_queue.get(timeout=self._parent_timeout,
block=True)
except Exception:
raise Empty
try:
if group not in pending_not_thread_safe_tests:
pending_tests.pop(group)
else:
pending_not_thread_safe_tests.pop(group)
except KeyError:
logger.debug('Got a result for unknown group: %s' % (group))
else:
completed_tests[group] = result
self._print_result(result)
if result.failures or result.errors:
failures += len(result.failures)
errors += len(result.errors)
if self.failfast:
# failfast is enabled, kill all the active workers
# and stop
for worker in workers:
if worker.is_alive():
worker.terminate()
break
except Empty:
worker_left = False
for worker in workers:
if worker.is_alive():
worker_left = True
break
if not worker_left:
break
# We are done, signalize all the workers to stop
stop_event.set()
end_time = time.time()
self._exit(start_time, end_time, failures, errors)
def __init__(self):
self.proc = None
self.daemon = None
self.stop = Event()
# Generate hash functions (as block masks)
params['masks'] = gen_masks(params['K'], params['T'], params['L'])
# Construct array of first T powers of 2 (in a reverse manner) for conversion between bit array and decimal values
params['bit_converter'] = 2**np.arange(params['T'])[::-1]
# Get numbre of samples
pmbr = ParrallelMacsBufferReader(params['data_file'], params['K'])
params['M'] = pmbr.get_data_shape()[0]
# Multiprocessing synchronization objects
producer_lock = Lock()
analyzer_lock = Lock()
# Event indicating that all hash producers are done with proessing current segment
all_producers_done_event = Event()
# Event indicating that all sketch analyzers are done
all_analyzers_done_event = Event()
# List of events indicating for every hash producer if it is idle (set) or working (clear).
free_producers_events = [Event() for i in range(params['num_producers'])]
for e in free_producers_events:
e.set()
# Manager for constructing shared-memory objects
manager = Manager()
# Dictionary holding shared-memory objects produced by server process manager
mpc_data = dict()
def test_enqueue(broker, admin_user):
broker.list_key = 'cluster_test:q'
broker.delete_queue()
a = async_task('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
hook='django_q.tests.test_cluster.assert_result',
broker=broker)
b = async_task('django_q.tests.tasks.count_letters2',
WordClass(),
hook='django_q.tests.test_cluster.assert_result',
broker=broker)
# unknown argument
c = async_task('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
'oneargumentoomany',
hook='django_q.tests.test_cluster.assert_bad_result',
broker=broker)
# unknown function
d = async_task('django_q.tests.tasks.does_not_exist',
WordClass(),
hook='django_q.tests.test_cluster.assert_bad_result',
broker=broker)
# function without result
e = async_task('django_q.tests.tasks.countdown', 100000, broker=broker)
# function as instance
f = async_task(multiply, 753, 2, hook=assert_result, broker=broker)
# model as argument
g = async_task('django_q.tests.tasks.get_task_name',
Task(name='John'),
broker=broker)
# args,kwargs, group and broken hook
h = async_task('django_q.tests.tasks.word_multiply',
2,
word='django',
hook='fail.me',
broker=broker)
# args unpickle test
j = async_task('django_q.tests.tasks.get_user_id',
admin_user,
broker=broker,
group='test_j')
# q_options and save opt_out test
k = async_task('django_q.tests.tasks.get_user_id',
admin_user,
q_options={
'broker': broker,
'group': 'test_k',
'save': False,
'timeout': 90
})
# test unicode
assert Task(name='Amalia').__unicode__() == 'Amalia'
# check if everything has a task id
assert isinstance(a, str)
assert isinstance(b, str)
assert isinstance(c, str)
assert isinstance(d, str)
assert isinstance(e, str)
assert isinstance(f, str)
assert isinstance(g, str)
assert isinstance(h, str)
assert isinstance(j, str)
assert isinstance(k, str)
# run the cluster to execute the tasks
task_count = 10
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push the tasks
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
# test wait timeout
assert result(j, wait=10) is None
assert fetch(j, wait=10) is None
assert result_group('test_j', wait=10) is None
assert result_group('test_j', count=2, wait=10) is None
assert fetch_group('test_j', wait=10) is None
assert fetch_group('test_j', count=2, wait=10) is None
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
# Check the results
# task a
result_a = fetch(a)
assert result_a is not None
assert result_a.success is True
assert result(a) == 1506
# task b
result_b = fetch(b)
assert result_b is not None
assert result_b.success is True
assert result(b) == 1506
# task c
result_c = fetch(c)
assert result_c is not None
assert result_c.success is False
# task d
result_d = fetch(d)
assert result_d is not None
assert result_d.success is False
# task e
result_e = fetch(e)
assert result_e is not None
assert result_e.success is True
assert result(e) is None
# task f
result_f = fetch(f)
assert result_f is not None
assert result_f.success is True
assert result(f) == 1506
# task g
result_g = fetch(g)
assert result_g is not None
assert result_g.success is True
assert result(g) == 'John'
# task h
result_h = fetch(h)
assert result_h is not None
assert result_h.success is True
assert result(h) == 12
# task j
result_j = fetch(j)
assert result_j is not None
assert result_j.success is True
assert result_j.result == result_j.args[0].id
# check fetch, result by name
assert fetch(result_j.name) == result_j
assert result(result_j.name) == result_j.result
# groups
assert result_group('test_j')[0] == result_j.result
assert result_j.group_result()[0] == result_j.result
assert result_group('test_j', failures=True)[0] == result_j.result
assert result_j.group_result(failures=True)[0] == result_j.result
assert fetch_group('test_j')[0].id == [result_j][0].id
assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id
assert count_group('test_j') == 1
assert result_j.group_count() == 1
assert count_group('test_j', failures=True) == 0
assert result_j.group_count(failures=True) == 0
assert delete_group('test_j') == 1
assert result_j.group_delete() == 0
deleted_group = delete_group('test_j', tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
deleted_group = result_j.group_delete(tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
# task k should not have been saved
assert fetch(k) is None
assert fetch(k, 100) is None
assert result(k, 100) is None
broker.delete_queue()
def __init__(self, workload_settings, target_settings, timer=None, *args):
self.ws = workload_settings
self.ts = target_settings
self.timer = timer and Timer(timer, self.abort) or None
self.shutdown_event = timer and Event() or None
self.worker_processes = []
startTime = time()
showInfo = False
noOutput = False
headless = False
vnc = False
for arg in argv:
if arg == '--info':
showInfo = True
if arg == '--nooutput':
noOutput = True
if arg == '--headless':
headless = True
if arg == '--vnc':
vnc = True
shutdownEvent = Event()
config = Config()
processes = []
console = Console(ouput=not noOutput)
driverManager = DriverManager(console, shutdownEvent)
driverVersion = driverManager.getDriverVersion('chrome')
browserVersion = driverManager.getBrowserVersion('chrome')
client = boto3.client('sqs')
totalMessageReceived = 0
stats = Stats()
messages = []
lastProcessStart = 0
lockThreadsCount = Lock()
def _showStats():
def test_double_download(self):
sd_hash_queue = Queue()
kill_event = Event()
dead_event = Event()
lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_event,
5209343)
uploader = Process(target=lbry_uploader.start)
uploader.start()
self.server_processes.append(uploader)
logging.debug("Testing double download")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = DummyRateLimiter()
sd_identifier = StreamDescriptorIdentifier()
downloaders = []
db_dir, blob_dir = mk_db_and_blob_dir()
self.session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=5553,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1],
external_ip="127.0.0.1")
self.stream_info_manager = DBEncryptedFileMetadataManager(
self.session.db_dir)
self.lbry_file_manager = EncryptedFileManager(self.session,
self.stream_info_manager,
sd_identifier)
@defer.inlineCallbacks
def make_downloader(metadata, prm):
info_validator = metadata.validator
options = metadata.options
factories = metadata.factories
chosen_options = [
o.default_value
for o in options.get_downloader_options(info_validator, prm)
]
downloader = yield factories[0].make_downloader(
metadata, chosen_options, prm)
defer.returnValue(downloader)
def append_downloader(downloader):
downloaders.append(downloader)
return downloader
@defer.inlineCallbacks
def download_file(sd_hash):
prm = self.session.payment_rate_manager
sd_blob = yield download_sd_blob(self.session, sd_hash, prm)
metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob)
downloader = yield make_downloader(metadata, prm)
downloaders.append(downloader)
finished_value = yield downloader.start()
defer.returnValue(finished_value)
def check_md5_sum():
f = open('test_file')
hashsum = MD5.new()
hashsum.update(f.read())
self.assertEqual(hashsum.hexdigest(),
"4ca2aafb4101c1e42235aad24fbb83be")
def delete_lbry_file():
logging.debug("deleting the file")
d = self.lbry_file_manager.delete_lbry_file(downloaders[0])
d.addCallback(
lambda _: self.lbry_file_manager.get_count_for_stream_hash(
downloaders[0].stream_hash))
d.addCallback(lambda c: self.stream_info_manager.delete_stream(
downloaders[1].stream_hash) if c == 0 else True)
return d
def check_lbry_file():
d = downloaders[1].status()
d.addCallback(lambda _: downloaders[1].status())
def check_status_report(status_report):
self.assertEqual(status_report.num_known,
status_report.num_completed)
self.assertEqual(status_report.num_known, 3)
d.addCallback(check_status_report)
return d
@defer.inlineCallbacks
def start_transfer(sd_hash):
logging.debug("Starting the transfer")
yield self.session.setup()
yield self.stream_info_manager.setup()
yield add_lbry_file_to_sd_identifier(sd_identifier)
yield self.lbry_file_manager.setup()
yield download_file(sd_hash)
yield check_md5_sum()
yield download_file(sd_hash)
yield check_lbry_file()
yield delete_lbry_file()
def stop(arg):
if isinstance(arg, Failure):
logging.debug("Client is stopping due to an error. Error: %s",
arg.getTraceback())
else:
logging.debug("Client is stopping normally.")
kill_event.set()
logging.debug("Set the kill event")
d = self.wait_for_event(dead_event, 15)
def print_shutting_down():
logging.info("Client is shutting down")
d.addCallback(lambda _: print_shutting_down())
d.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir))
d.addCallback(lambda _: arg)
return d
d = self.wait_for_hash_from_queue(sd_hash_queue)
d.addCallback(start_transfer)
d.addBoth(stop)
return d
for q in queues:
q.put(w)
if __name__ == "__main__":
buffer_size = int(1e+6)
env_dict = {"obs": {"shape": 4},
"act": {},
"rew": {},
"next_obs": {"shape": 4},
"done": {}}
n_explorer = 4
global_rb = MPPrioritizedReplayBuffer(buffer_size,env_dict)
is_training_done = Event()
is_training_done.clear()
qs = [SimpleQueue() for _ in range(n_explorer)]
ps = [Process(target=explorer,
args=[global_rb,env_dict,is_training_done,q])
for q in qs]
for p in ps:
p.start()
learner(global_rb,qs)
is_training_done.set()
for p in ps:
p.join()
process_read = []
#CL-> Explain how each of the different list works together
'''
As Multiprocessing creates different processes, the CPU will allocate a different sector of memory to each process
With each having a different sector of memory, sharing of data in that situation is not possible
With use of Managers, it creates a global shared memory between processes which all processes are able to have reference to
'''
manager = Manager()
initial_id = manager.list() # Store Initial camID before replacement
r_id = manager.list() # Store camID to be replaced to (with reference to camera 1)
cam_id = manager.list() # Store camera camID of detected re-identifications
unique_id = manager.list() # Store list of Unique IDs
# Flag to indicate that all main detections processes have terminated, and indicate re-id process to end.
# Else it'll be a zombie running in background
flag = Event()
rp = Process(target=extract_query,args=(initial_id,r_id,cam_id,flag,unique_id))
#Run up reid process only when more than 1 camera source is inputted
if len(data) > 1: # Only run re-identification if more than 1 camera source declared (Obj 3)
rp.start()
#Declare Queue objects and Processes (Obj 1)
#For each camera, two separate process will be forked to handle queue and detection
#"Process capture" : Process to handle the ingestion of the camera frames into buffer
#"Process read" : Process to handle the retrieval of the buffer and then perform detection
for i in range(len(data)):
queue_name = 'processes_'+str(processes)
queue_name = manager.Queue()
# Queue first, then detection (Obj 2)
def setup(self):
self.root = RootTask(should_stop=Event(), should_pause=Event())
self.task = ConditionalTask(name='Test')
self.root.add_child_task(0, self.task)
self.check = CheckTask(name='check')
self.task.add_child_task(0, self.check)
contentWeight = 6
relationWeight = 6
##########################
#
# # In[4]:
#
# print len(datasets)
# extracted_graph = extract(datasets)
# # with open("/Users/jimmccusker/Dropbox/Public/ecstra-unstemmed.ttl",'wb') as out:
# with open("/home/jason/Documents/TWC/linkipedia/output_file/ecstra-unstemmed-dataone-0318.ttl",'wb') as out:
# out.write(extracted_graph.serialize(format='turtle'))
NUMBER_OF_PROCESSES = 6
processed = Event()
def work(id, jobs, result):
while True:
try:
dataset = jobs.get(timeout=10)
except Empty:
processed.set()
break
try:
matched_classes = collections.defaultdict(set)
r = set()
print dataset
#abstract = get_dataset_abstract(dataset)
attributes = get_attributes(dataset)
def unit_test():
global TEST
TEST=True
VULBOX_IP = '127.0.0.1'
TEST_PORT = 5342
s = Scheduler(TEAM_LIST_PATH)
s.update_status()
# time.sleep(1)
s.update_state(TEST_GETSTATE)
print 'teams'
print s.teams
print 'services'
print s.services
print 'scripts'
print s.scripts
print 'run_list'
print s.run_list
s.update_status()
#time.sleep(1)
tid = 20
script_id = 2
service_id = 1
s.teams = {1:{"team_id":1,"ip":'127.0.12.2'},tid:{"team_id":tid,"ip":'127.0.13.2'}}
gs = copy.deepcopy(TEST_GETSTATE)
gs['state_expire']=60
gs['run_scripts'].append( {"team_id":tid,"run_list":[1,script_id,3]})
s.update_state(gs)
print ''
print s.get_rand_delay(s.run_list[tid])
tid = 1
sid =1
srvid=1
#test script run
print 'trying to run script'
p = os.path.join(os.getcwd(),'test','scripts','20','2','2.py')
try:
s.runscript(Event(),tid,sid,srvid,300,'setflag',
p,VULBOX_IP,TEST_PORT,0.01)
except Exception as e:
print 'Exception '+str(e)
s.update_status()
#time.sleep(1)
#test script
print 'testing bundle script run'
is_bundle = 1
fn = s.get_script_repo_path(sid)
print 'Repo path '+fn
if os.path.exists(fn):
print fn+'exits'
os.remove(fn)
s.update_script_repo(sid)
assert os.path.exists(fn)
print 'repo updated'
fn_tgz = s.get_script_path(tid,sid,is_bundle)
fn_f = os.path.join(os.path.dirname(fn_tgz),'exploit.py')
print 'Script path '+str((fn_f,fn_tgz))
if os.path.exists(fn_f):
print fn_f+'exits'
os.remove(fn_f)
if os.path.exists(fn_tgz):
print fn_tgz+'exits'
os.remove(fn_tgz)
print 'updating script'
fn = s.update_script(tid,sid,is_bundle)
assert fn == fn_f
assert os.path.exists(fn_f)
assert os.path.exists(fn_tgz)
print 'Bundled script extraction OK'
print 'Trying to run'
try:
s.runscript(Event(),tid,sid,srvid,300,'exploit',
fn,'127.0.2.1', TEST_PORT,0.1)
except Exception as e:
print 'Exception '+str(e)
exit(0)
# s.schedule_scripts()
c = 0
while True:
s.clean_process()
s.update_status()
c+=1
if c==5:
s.kill_process()
time.sleep(1)
return 0
contracts = contracts[args.skip:]
log("Setting up workers.")
# Set up multiprocessing result list and queue.
manager = Manager()
# This list contains analysis results as
# (filename, category, meta, analytics) quadruples.
res_list = manager.list()
# Holds results transiently before flushing to res_list
res_queue = SimpleQueue()
# Start the periodic flush process, only run while run_signal is set.
run_signal = Event()
run_signal.set()
flush_proc = Process(target=flush_queue,
args=(run_signal, res_queue, res_list))
flush_proc.start()
workers = []
avail_jobs = list(range(args.jobs))
contract_iter = enumerate(contracts)
contracts_exhausted = False
log("Analysing...\n")
try:
while not contracts_exhausted:
# If there's both workers and contracts available, use the former to work on the latter.
while not contracts_exhausted and len(avail_jobs) > 0:
def __init__(
self,
exp_dir,
policy_pickle,
env_pickle,
baseline_pickle,
dynamics_model_pickle,
feed_dicts,
n_itr,
num_inner_grad_steps,
flags_need_query,
num_rollouts_per_iter,
config,
simulation_sleep,
initial_random_samples=True,
start_itr=0,
sampler_str='mbmpo',
):
self.initial_random_samples = initial_random_samples
worker_instances = [
WorkerData(
num_rollouts_per_iter=num_rollouts_per_iter,
simulation_sleep=simulation_sleep,
),
WorkerModel(),
WorkerPolicy(num_inner_grad_steps=num_inner_grad_steps,
sampler_str=sampler_str),
]
names = ["Data", "Model", "Policy"]
# one queue for each worker, tasks assigned by scheduler and previous worker
queues = [Queue(-1) for _ in range(3)]
# worker sends task-completed notification and time info to scheduler
worker_remotes, remotes = zip(*[Pipe() for _ in range(3)])
# stop condition
stop_cond = Event()
# current worker needs query means previous workers does not auto push
# skipped checking here
flags_auto_push = [
not flags_need_query[1], not flags_need_query[2],
not flags_need_query[0]
]
self.ps = [
Process(
target=worker_instance,
name=name,
args=(
exp_dir,
policy_pickle,
env_pickle,
baseline_pickle,
dynamics_model_pickle,
feed_dict,
queue_prev,
queue,
queue_next,
worker_remote,
start_itr,
n_itr,
stop_cond,
need_query,
auto_push,
config,
),
) for (worker_instance, name, feed_dict, queue_prev, queue,
queue_next, worker_remote, need_query, auto_push) in zip(
worker_instances,
names,
feed_dicts,
queues[2:] + queues[:2],
queues,
queues[1:] + queues[:1],
worker_remotes,
flags_need_query,
flags_auto_push,
)
]
# central scheduler sends command and receives receipts
self.names = names
self.queues = queues
self.remotes = remotes
'cap' + str(work) + '.pcap'
],
stdout=subprocess.PIPE)
print("Stopping tcpcapture ...")
p.send_signal(subprocess.signal.SIGTERM)
print("tcpdump completed")
# Save a reference to the original signal handler for SIGINT.
default_handler = signal.getsignal(signal.SIGINT)
# Set signal handling of SIGINT to ignore mode.
signal.signal(signal.SIGINT, signal.SIG_IGN)
exit_event = Event()
work_queue = Queue()
print("exit_event - " + str(exit_event.is_set()))
# Spawn the worker process.
cp = Process(
target=tcpdump_capture,
args=(exit_event, work_queue),
)
cp.start()
# Since we spawned all the necessary processes already,
# restore default signal handling for the parent process.
signal.signal(signal.SIGINT, default_handler)
def __init__(self, pid=None):
pid = pid or os.getpid()
self.start_mem = psutil.Process(pid).memory_info().rss
self.event = Event()
self.p = MemoryTrackingProcess(pid, self.event)
def setup(self):
self.root = RootTask(should_stop=Event(), should_pause=Event())
self.task = WhileTask(task_name='Test')
self.root.children_task.append(self.task)
self.check = CheckTask(task_name='check')
self.task.children_task.append(self.check)
def _setup_comunication(self):
self._server_closed = Event()
process = Process(target=self._gather_subscriptions,
args=(self._server_closed, ),
daemon=True)
process.start()
